2024-12-10 07:26:13,965 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-10 07:26:13,979 main DEBUG Took 0.011750 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 07:26:13,980 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 07:26:13,980 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 07:26:13,981 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 07:26:13,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:13,990 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 07:26:14,006 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,008 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,009 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,011 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,012 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,013 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,014 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,015 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,016 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,017 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,018 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,019 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:26:14,020 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,020 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 07:26:14,022 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:26:14,023 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 07:26:14,026 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 07:26:14,026 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 07:26:14,028 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 07:26:14,029 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 07:26:14,036 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 07:26:14,039 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 07:26:14,040 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 07:26:14,041 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 07:26:14,041 main DEBUG createAppenders(={Console}) 2024-12-10 07:26:14,042 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-12-10 07:26:14,042 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-10 07:26:14,042 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-12-10 07:26:14,043 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 07:26:14,043 main DEBUG OutputStream closed 2024-12-10 07:26:14,044 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 07:26:14,044 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 07:26:14,044 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-12-10 07:26:14,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 07:26:14,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 07:26:14,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 07:26:14,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 07:26:14,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 07:26:14,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 07:26:14,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 07:26:14,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 07:26:14,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 07:26:14,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 07:26:14,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 07:26:14,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 07:26:14,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 07:26:14,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 07:26:14,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 07:26:14,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 07:26:14,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 07:26:14,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 07:26:14,126 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 07:26:14,127 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-12-10 07:26:14,127 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 07:26:14,128 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-12-10T07:26:14,144 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-12-10 07:26:14,147 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 07:26:14,147 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T07:26:14,203 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=79, ProcessCount=11, AvailableMemoryMB=5513 2024-12-10T07:26:14,464 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2 2024-12-10T07:26:14,998 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T07:26:15,101 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T07:26:15,146 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-12-10T07:26:15,188 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575179 2024-12-10T07:26:15,243 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575179 2024-12-10T07:26:15,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,310 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-12-10T07:26:15,310 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-12-10T07:26:15,340 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 493d0679b9ca3cdf33d820337051847c, NAME => 'testLockupAroundBadAssignSync,,1733815575321.493d0679b9ca3cdf33d820337051847c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2 2024-12-10T07:26:15,377 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T07:26:15,380 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1733815575321.493d0679b9ca3cdf33d820337051847c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:26:15,446 INFO [StoreOpener-493d0679b9ca3cdf33d820337051847c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region 493d0679b9ca3cdf33d820337051847c 2024-12-10T07:26:15,473 INFO [StoreOpener-493d0679b9ca3cdf33d820337051847c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 493d0679b9ca3cdf33d820337051847c columnFamilyName MyCF 2024-12-10T07:26:15,478 DEBUG [StoreOpener-493d0679b9ca3cdf33d820337051847c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:26:15,482 INFO [StoreOpener-493d0679b9ca3cdf33d820337051847c-1 {}] regionserver.HStore(327): Store=493d0679b9ca3cdf33d820337051847c/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:26:15,485 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 493d0679b9ca3cdf33d820337051847c 2024-12-10T07:26:15,489 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/data/default/testLockupAroundBadAssignSync/493d0679b9ca3cdf33d820337051847c 2024-12-10T07:26:15,490 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/data/default/testLockupAroundBadAssignSync/493d0679b9ca3cdf33d820337051847c 2024-12-10T07:26:15,492 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 493d0679b9ca3cdf33d820337051847c 2024-12-10T07:26:15,493 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 493d0679b9ca3cdf33d820337051847c 2024-12-10T07:26:15,498 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 493d0679b9ca3cdf33d820337051847c 2024-12-10T07:26:15,507 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/data/default/testLockupAroundBadAssignSync/493d0679b9ca3cdf33d820337051847c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:26:15,508 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 493d0679b9ca3cdf33d820337051847c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70352700, jitterRate=0.048336923122406006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T07:26:15,515 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T07:26:15,529 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 493d0679b9ca3cdf33d820337051847c: Writing region info on filesystem at 1733815575420Initializing all the Stores at 1733815575421 (+1 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1733815575422 (+1 ms)Cleaning up temporary data from old regions at 1733815575493 (+71 ms)Region opened successfully at 1733815575526 (+33 ms) 2024-12-10T07:26:15,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575179) roll requested 2024-12-10T07:26:15,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575579 2024-12-10T07:26:15,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575179 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575579 2024-12-10T07:26:15,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575579) roll requested 2024-12-10T07:26:15,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575592 2024-12-10T07:26:15,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575592 2024-12-10T07:26:15,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575592) roll requested 2024-12-10T07:26:15,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575606 2024-12-10T07:26:15,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575606 2024-12-10T07:26:15,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575606) roll requested 2024-12-10T07:26:15,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575618 2024-12-10T07:26:15,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575618 2024-12-10T07:26:15,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575618) roll requested 2024-12-10T07:26:15,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575629 2024-12-10T07:26:15,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575629 2024-12-10T07:26:15,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575629) roll requested 2024-12-10T07:26:15,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575642 2024-12-10T07:26:15,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575642 2024-12-10T07:26:15,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575642) roll requested 2024-12-10T07:26:15,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575652 2024-12-10T07:26:15,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575652 2024-12-10T07:26:15,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575652) roll requested 2024-12-10T07:26:15,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575663 2024-12-10T07:26:15,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575663 2024-12-10T07:26:15,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575663) roll requested 2024-12-10T07:26:15,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575673 2024-12-10T07:26:15,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575673 2024-12-10T07:26:15,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575673) roll requested 2024-12-10T07:26:15,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575683 2024-12-10T07:26:15,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575683 2024-12-10T07:26:15,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575683) roll requested 2024-12-10T07:26:15,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575693 2024-12-10T07:26:15,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575693 2024-12-10T07:26:15,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575693) roll requested 2024-12-10T07:26:15,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575702 2024-12-10T07:26:15,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575702 2024-12-10T07:26:15,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575702) roll requested 2024-12-10T07:26:15,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575711 2024-12-10T07:26:15,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575711 2024-12-10T07:26:15,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575711) roll requested 2024-12-10T07:26:15,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575720 2024-12-10T07:26:15,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575720 2024-12-10T07:26:15,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575720) roll requested 2024-12-10T07:26:15,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575730 2024-12-10T07:26:15,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575730 2024-12-10T07:26:15,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575730) roll requested 2024-12-10T07:26:15,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575740 2024-12-10T07:26:15,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575740 2024-12-10T07:26:15,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575740) roll requested 2024-12-10T07:26:15,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575749 2024-12-10T07:26:15,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575749 2024-12-10T07:26:15,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575749) roll requested 2024-12-10T07:26:15,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575760 2024-12-10T07:26:15,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575760 2024-12-10T07:26:15,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575760) roll requested 2024-12-10T07:26:15,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575768 2024-12-10T07:26:15,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575768 2024-12-10T07:26:15,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575768) roll requested 2024-12-10T07:26:15,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575778 2024-12-10T07:26:15,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575778 2024-12-10T07:26:15,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575778) roll requested 2024-12-10T07:26:15,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575790 2024-12-10T07:26:15,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575790 2024-12-10T07:26:15,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575790) roll requested 2024-12-10T07:26:15,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575805 2024-12-10T07:26:15,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575805 2024-12-10T07:26:15,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575805) roll requested 2024-12-10T07:26:15,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575813 2024-12-10T07:26:15,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575813 2024-12-10T07:26:15,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575813) roll requested 2024-12-10T07:26:15,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575825 2024-12-10T07:26:15,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575825 2024-12-10T07:26:15,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575825) roll requested 2024-12-10T07:26:15,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575837 2024-12-10T07:26:15,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575837 2024-12-10T07:26:15,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575837) roll requested 2024-12-10T07:26:15,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575844 2024-12-10T07:26:15,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575844 2024-12-10T07:26:15,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575844) roll requested 2024-12-10T07:26:15,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575852 2024-12-10T07:26:15,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575852 2024-12-10T07:26:15,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575852) roll requested 2024-12-10T07:26:15,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575859 2024-12-10T07:26:15,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575859 2024-12-10T07:26:15,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575859) roll requested 2024-12-10T07:26:15,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575869 2024-12-10T07:26:15,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575869 2024-12-10T07:26:15,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575869) roll requested 2024-12-10T07:26:15,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575880 2024-12-10T07:26:15,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575880 2024-12-10T07:26:15,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575880) roll requested 2024-12-10T07:26:15,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575889 2024-12-10T07:26:15,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575889 2024-12-10T07:26:15,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575889) roll requested 2024-12-10T07:26:15,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575898 2024-12-10T07:26:15,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575898 2024-12-10T07:26:15,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575898) roll requested 2024-12-10T07:26:15,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575907 2024-12-10T07:26:15,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575907 2024-12-10T07:26:15,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575907) roll requested 2024-12-10T07:26:15,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575916 2024-12-10T07:26:15,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575916 2024-12-10T07:26:15,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575916) roll requested 2024-12-10T07:26:15,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575925 2024-12-10T07:26:15,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575925 2024-12-10T07:26:15,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575925) roll requested 2024-12-10T07:26:15,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575935 2024-12-10T07:26:15,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575935 2024-12-10T07:26:15,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575935) roll requested 2024-12-10T07:26:15,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575946 2024-12-10T07:26:15,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575946 2024-12-10T07:26:15,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575946) roll requested 2024-12-10T07:26:15,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575955 2024-12-10T07:26:15,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575955 2024-12-10T07:26:15,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575955) roll requested 2024-12-10T07:26:15,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575964 2024-12-10T07:26:15,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575964 2024-12-10T07:26:15,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575964) roll requested 2024-12-10T07:26:15,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575974 2024-12-10T07:26:15,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575974 2024-12-10T07:26:15,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575974) roll requested 2024-12-10T07:26:15,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575982 2024-12-10T07:26:15,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575982 2024-12-10T07:26:15,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:15,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:15,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575982) roll requested 2024-12-10T07:26:15,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815575990 2024-12-10T07:26:15,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:15,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575990 2024-12-10T07:26:16,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815575990) roll requested 2024-12-10T07:26:16,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576001 2024-12-10T07:26:16,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815575990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576001 2024-12-10T07:26:16,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576001) roll requested 2024-12-10T07:26:16,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576014 2024-12-10T07:26:16,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576014 2024-12-10T07:26:16,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576014) roll requested 2024-12-10T07:26:16,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576024 2024-12-10T07:26:16,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576024 2024-12-10T07:26:16,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576024) roll requested 2024-12-10T07:26:16,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576031 2024-12-10T07:26:16,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576031 2024-12-10T07:26:16,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576031) roll requested 2024-12-10T07:26:16,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576038 2024-12-10T07:26:16,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576038 2024-12-10T07:26:16,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576038) roll requested 2024-12-10T07:26:16,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576045 2024-12-10T07:26:16,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576045 2024-12-10T07:26:16,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576045) roll requested 2024-12-10T07:26:16,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576053 2024-12-10T07:26:16,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576053 2024-12-10T07:26:16,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576053) roll requested 2024-12-10T07:26:16,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576066 2024-12-10T07:26:16,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576066 2024-12-10T07:26:16,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576066) roll requested 2024-12-10T07:26:16,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576074 2024-12-10T07:26:16,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576074 2024-12-10T07:26:16,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576074) roll requested 2024-12-10T07:26:16,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576082 2024-12-10T07:26:16,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576082 2024-12-10T07:26:16,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576082) roll requested 2024-12-10T07:26:16,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576089 2024-12-10T07:26:16,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576089 2024-12-10T07:26:16,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576089) roll requested 2024-12-10T07:26:16,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576096 2024-12-10T07:26:16,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576096 2024-12-10T07:26:16,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576096) roll requested 2024-12-10T07:26:16,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576104 2024-12-10T07:26:16,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576104 2024-12-10T07:26:16,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576104) roll requested 2024-12-10T07:26:16,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576113 2024-12-10T07:26:16,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576113 2024-12-10T07:26:16,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576113) roll requested 2024-12-10T07:26:16,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576125 2024-12-10T07:26:16,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576125 2024-12-10T07:26:16,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576125) roll requested 2024-12-10T07:26:16,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576139 2024-12-10T07:26:16,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576139 2024-12-10T07:26:16,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576139) roll requested 2024-12-10T07:26:16,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576145 2024-12-10T07:26:16,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576145 2024-12-10T07:26:16,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576145) roll requested 2024-12-10T07:26:16,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576152 2024-12-10T07:26:16,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576152 2024-12-10T07:26:16,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576152) roll requested 2024-12-10T07:26:16,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576160 2024-12-10T07:26:16,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576160 2024-12-10T07:26:16,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576160) roll requested 2024-12-10T07:26:16,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576169 2024-12-10T07:26:16,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576169 2024-12-10T07:26:16,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576169) roll requested 2024-12-10T07:26:16,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576178 2024-12-10T07:26:16,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576178 2024-12-10T07:26:16,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576178) roll requested 2024-12-10T07:26:16,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576201 2024-12-10T07:26:16,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576201 2024-12-10T07:26:16,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576201) roll requested 2024-12-10T07:26:16,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576209 2024-12-10T07:26:16,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576209 2024-12-10T07:26:16,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576209) roll requested 2024-12-10T07:26:16,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576216 2024-12-10T07:26:16,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576216 2024-12-10T07:26:16,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576216) roll requested 2024-12-10T07:26:16,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576222 2024-12-10T07:26:16,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576222 2024-12-10T07:26:16,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576222) roll requested 2024-12-10T07:26:16,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576228 2024-12-10T07:26:16,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576228 2024-12-10T07:26:16,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576228) roll requested 2024-12-10T07:26:16,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576237 2024-12-10T07:26:16,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576237 2024-12-10T07:26:16,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576237) roll requested 2024-12-10T07:26:16,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576245 2024-12-10T07:26:16,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576245 2024-12-10T07:26:16,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576245) roll requested 2024-12-10T07:26:16,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576250 2024-12-10T07:26:16,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576250 2024-12-10T07:26:16,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576250) roll requested 2024-12-10T07:26:16,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576257 2024-12-10T07:26:16,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576257 2024-12-10T07:26:16,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576257) roll requested 2024-12-10T07:26:16,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576266 2024-12-10T07:26:16,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576266 2024-12-10T07:26:16,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576266) roll requested 2024-12-10T07:26:16,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576275 2024-12-10T07:26:16,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576275 2024-12-10T07:26:16,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576275) roll requested 2024-12-10T07:26:16,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576283 2024-12-10T07:26:16,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576283 2024-12-10T07:26:16,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576283) roll requested 2024-12-10T07:26:16,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576293 2024-12-10T07:26:16,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576293 2024-12-10T07:26:16,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576293) roll requested 2024-12-10T07:26:16,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576303 2024-12-10T07:26:16,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576303 2024-12-10T07:26:16,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576303) roll requested 2024-12-10T07:26:16,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576313 2024-12-10T07:26:16,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576313 2024-12-10T07:26:16,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576313) roll requested 2024-12-10T07:26:16,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576321 2024-12-10T07:26:16,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576321 2024-12-10T07:26:16,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576321) roll requested 2024-12-10T07:26:16,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576329 2024-12-10T07:26:16,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576329 2024-12-10T07:26:16,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576329) roll requested 2024-12-10T07:26:16,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576338 2024-12-10T07:26:16,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576338 2024-12-10T07:26:16,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576338) roll requested 2024-12-10T07:26:16,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576345 2024-12-10T07:26:16,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576345 2024-12-10T07:26:16,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576345) roll requested 2024-12-10T07:26:16,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576353 2024-12-10T07:26:16,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576353 2024-12-10T07:26:16,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576353) roll requested 2024-12-10T07:26:16,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576362 2024-12-10T07:26:16,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576362 2024-12-10T07:26:16,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576362) roll requested 2024-12-10T07:26:16,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576373 2024-12-10T07:26:16,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576373 2024-12-10T07:26:16,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576373) roll requested 2024-12-10T07:26:16,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576379 2024-12-10T07:26:16,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576379 2024-12-10T07:26:16,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576379) roll requested 2024-12-10T07:26:16,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576387 2024-12-10T07:26:16,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576387 2024-12-10T07:26:16,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576387) roll requested 2024-12-10T07:26:16,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576395 2024-12-10T07:26:16,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576395 2024-12-10T07:26:16,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576395) roll requested 2024-12-10T07:26:16,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576403 2024-12-10T07:26:16,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576403 2024-12-10T07:26:16,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576403) roll requested 2024-12-10T07:26:16,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576413 2024-12-10T07:26:16,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576413 2024-12-10T07:26:16,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576413) roll requested 2024-12-10T07:26:16,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576421 2024-12-10T07:26:16,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576421 2024-12-10T07:26:16,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576421) roll requested 2024-12-10T07:26:16,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576428 2024-12-10T07:26:16,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576428 2024-12-10T07:26:16,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576428) roll requested 2024-12-10T07:26:16,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576439 2024-12-10T07:26:16,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576439 2024-12-10T07:26:16,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576439) roll requested 2024-12-10T07:26:16,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576448 2024-12-10T07:26:16,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576448 2024-12-10T07:26:16,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576448) roll requested 2024-12-10T07:26:16,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576456 2024-12-10T07:26:16,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576456 2024-12-10T07:26:16,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576456) roll requested 2024-12-10T07:26:16,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576465 2024-12-10T07:26:16,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576465 2024-12-10T07:26:16,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576465) roll requested 2024-12-10T07:26:16,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576473 2024-12-10T07:26:16,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576473 2024-12-10T07:26:16,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576473) roll requested 2024-12-10T07:26:16,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576481 2024-12-10T07:26:16,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576481 2024-12-10T07:26:16,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576481) roll requested 2024-12-10T07:26:16,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576490 2024-12-10T07:26:16,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576490 2024-12-10T07:26:16,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576490) roll requested 2024-12-10T07:26:16,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576501 2024-12-10T07:26:16,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576501 2024-12-10T07:26:16,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576501) roll requested 2024-12-10T07:26:16,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576507 2024-12-10T07:26:16,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576507 2024-12-10T07:26:16,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576507) roll requested 2024-12-10T07:26:16,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576515 2024-12-10T07:26:16,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576515 2024-12-10T07:26:16,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576515) roll requested 2024-12-10T07:26:16,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576522 2024-12-10T07:26:16,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576522 2024-12-10T07:26:16,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576522) roll requested 2024-12-10T07:26:16,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576531 2024-12-10T07:26:16,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576531 2024-12-10T07:26:16,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576531) roll requested 2024-12-10T07:26:16,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576539 2024-12-10T07:26:16,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576539 2024-12-10T07:26:16,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576539) roll requested 2024-12-10T07:26:16,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576549 2024-12-10T07:26:16,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576549 2024-12-10T07:26:16,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576549) roll requested 2024-12-10T07:26:16,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576559 2024-12-10T07:26:16,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576559 2024-12-10T07:26:16,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576559) roll requested 2024-12-10T07:26:16,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576569 2024-12-10T07:26:16,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576569 2024-12-10T07:26:16,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576569) roll requested 2024-12-10T07:26:16,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576580 2024-12-10T07:26:16,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576580 2024-12-10T07:26:16,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576580) roll requested 2024-12-10T07:26:16,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576590 2024-12-10T07:26:16,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576590 2024-12-10T07:26:16,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576590) roll requested 2024-12-10T07:26:16,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576601 2024-12-10T07:26:16,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576601 2024-12-10T07:26:16,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576601) roll requested 2024-12-10T07:26:16,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576610 2024-12-10T07:26:16,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576610 2024-12-10T07:26:16,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576610) roll requested 2024-12-10T07:26:16,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576619 2024-12-10T07:26:16,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576619 2024-12-10T07:26:16,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576619) roll requested 2024-12-10T07:26:16,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576628 2024-12-10T07:26:16,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576628 2024-12-10T07:26:16,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576628) roll requested 2024-12-10T07:26:16,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576638 2024-12-10T07:26:16,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576638 2024-12-10T07:26:16,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576638) roll requested 2024-12-10T07:26:16,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576648 2024-12-10T07:26:16,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576648 2024-12-10T07:26:16,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576648) roll requested 2024-12-10T07:26:16,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576657 2024-12-10T07:26:16,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576657 2024-12-10T07:26:16,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576657) roll requested 2024-12-10T07:26:16,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576666 2024-12-10T07:26:16,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576666 2024-12-10T07:26:16,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576666) roll requested 2024-12-10T07:26:16,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576673 2024-12-10T07:26:16,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576673 2024-12-10T07:26:16,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576673) roll requested 2024-12-10T07:26:16,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576680 2024-12-10T07:26:16,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576680 2024-12-10T07:26:16,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576680) roll requested 2024-12-10T07:26:16,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576688 2024-12-10T07:26:16,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576688 2024-12-10T07:26:16,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576688) roll requested 2024-12-10T07:26:16,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576698 2024-12-10T07:26:16,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576698 2024-12-10T07:26:16,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576698) roll requested 2024-12-10T07:26:16,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576707 2024-12-10T07:26:16,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576707 2024-12-10T07:26:16,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576707) roll requested 2024-12-10T07:26:16,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576718 2024-12-10T07:26:16,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576718 2024-12-10T07:26:16,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576718) roll requested 2024-12-10T07:26:16,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576729 2024-12-10T07:26:16,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576729 2024-12-10T07:26:16,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576729) roll requested 2024-12-10T07:26:16,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576739 2024-12-10T07:26:16,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576739 2024-12-10T07:26:16,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576739) roll requested 2024-12-10T07:26:16,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576746 2024-12-10T07:26:16,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576746 2024-12-10T07:26:16,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576746) roll requested 2024-12-10T07:26:16,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576754 2024-12-10T07:26:16,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576754 2024-12-10T07:26:16,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576754) roll requested 2024-12-10T07:26:16,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576762 2024-12-10T07:26:16,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576762 2024-12-10T07:26:16,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576762) roll requested 2024-12-10T07:26:16,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576770 2024-12-10T07:26:16,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576770 2024-12-10T07:26:16,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576770) roll requested 2024-12-10T07:26:16,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576780 2024-12-10T07:26:16,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576780 2024-12-10T07:26:16,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576780) roll requested 2024-12-10T07:26:16,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576790 2024-12-10T07:26:16,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576790 2024-12-10T07:26:16,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576790) roll requested 2024-12-10T07:26:16,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576796 2024-12-10T07:26:16,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576796 2024-12-10T07:26:16,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576796) roll requested 2024-12-10T07:26:16,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576804 2024-12-10T07:26:16,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576804 2024-12-10T07:26:16,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576804) roll requested 2024-12-10T07:26:16,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576811 2024-12-10T07:26:16,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576811 2024-12-10T07:26:16,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576811) roll requested 2024-12-10T07:26:16,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576822 2024-12-10T07:26:16,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576822 2024-12-10T07:26:16,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576822) roll requested 2024-12-10T07:26:16,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576832 2024-12-10T07:26:16,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576832 2024-12-10T07:26:16,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576832) roll requested 2024-12-10T07:26:16,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576841 2024-12-10T07:26:16,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576841 2024-12-10T07:26:16,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576841) roll requested 2024-12-10T07:26:16,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576847 2024-12-10T07:26:16,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576847 2024-12-10T07:26:16,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576847) roll requested 2024-12-10T07:26:16,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576855 2024-12-10T07:26:16,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576855 2024-12-10T07:26:16,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576855) roll requested 2024-12-10T07:26:16,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576863 2024-12-10T07:26:16,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576863 2024-12-10T07:26:16,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576863) roll requested 2024-12-10T07:26:16,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576876 2024-12-10T07:26:16,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576876 2024-12-10T07:26:16,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576876) roll requested 2024-12-10T07:26:16,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576885 2024-12-10T07:26:16,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576885 2024-12-10T07:26:16,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576885) roll requested 2024-12-10T07:26:16,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576893 2024-12-10T07:26:16,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576893 2024-12-10T07:26:16,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576893) roll requested 2024-12-10T07:26:16,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576901 2024-12-10T07:26:16,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576901 2024-12-10T07:26:16,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576901) roll requested 2024-12-10T07:26:16,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576912 2024-12-10T07:26:16,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576912 2024-12-10T07:26:16,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576912) roll requested 2024-12-10T07:26:16,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576918 2024-12-10T07:26:16,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576918 2024-12-10T07:26:16,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576918) roll requested 2024-12-10T07:26:16,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576925 2024-12-10T07:26:16,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576925 2024-12-10T07:26:16,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576925) roll requested 2024-12-10T07:26:16,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576931 2024-12-10T07:26:16,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576931 2024-12-10T07:26:16,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576931) roll requested 2024-12-10T07:26:16,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576938 2024-12-10T07:26:16,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576938 2024-12-10T07:26:16,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576938) roll requested 2024-12-10T07:26:16,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576945 2024-12-10T07:26:16,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576945 2024-12-10T07:26:16,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576945) roll requested 2024-12-10T07:26:16,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576952 2024-12-10T07:26:16,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576952 2024-12-10T07:26:16,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576952) roll requested 2024-12-10T07:26:16,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576958 2024-12-10T07:26:16,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576958 2024-12-10T07:26:16,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576958) roll requested 2024-12-10T07:26:16,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576969 2024-12-10T07:26:16,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576969 2024-12-10T07:26:16,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576969) roll requested 2024-12-10T07:26:16,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576976 2024-12-10T07:26:16,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576976 2024-12-10T07:26:16,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576976) roll requested 2024-12-10T07:26:16,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576985 2024-12-10T07:26:16,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576985 2024-12-10T07:26:16,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576985) roll requested 2024-12-10T07:26:16,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576992 2024-12-10T07:26:16,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:16,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576992 2024-12-10T07:26:16,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:16,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:16,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576992) roll requested 2024-12-10T07:26:16,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815576999 2024-12-10T07:26:17,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576999 2024-12-10T07:26:17,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815576999) roll requested 2024-12-10T07:26:17,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577005 2024-12-10T07:26:17,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815576999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577005 2024-12-10T07:26:17,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577005) roll requested 2024-12-10T07:26:17,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577013 2024-12-10T07:26:17,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577013 2024-12-10T07:26:17,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577013) roll requested 2024-12-10T07:26:17,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577021 2024-12-10T07:26:17,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577021 2024-12-10T07:26:17,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577021) roll requested 2024-12-10T07:26:17,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577028 2024-12-10T07:26:17,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577028 2024-12-10T07:26:17,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577028) roll requested 2024-12-10T07:26:17,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577040 2024-12-10T07:26:17,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577040 2024-12-10T07:26:17,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577040) roll requested 2024-12-10T07:26:17,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577049 2024-12-10T07:26:17,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577049 2024-12-10T07:26:17,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577049) roll requested 2024-12-10T07:26:17,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577060 2024-12-10T07:26:17,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577060 2024-12-10T07:26:17,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577060) roll requested 2024-12-10T07:26:17,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577067 2024-12-10T07:26:17,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577067 2024-12-10T07:26:17,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577067) roll requested 2024-12-10T07:26:17,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577073 2024-12-10T07:26:17,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577073 2024-12-10T07:26:17,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577073) roll requested 2024-12-10T07:26:17,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577081 2024-12-10T07:26:17,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577081 2024-12-10T07:26:17,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577081) roll requested 2024-12-10T07:26:17,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577092 2024-12-10T07:26:17,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577092 2024-12-10T07:26:17,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577092) roll requested 2024-12-10T07:26:17,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577099 2024-12-10T07:26:17,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577099 2024-12-10T07:26:17,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577099) roll requested 2024-12-10T07:26:17,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577106 2024-12-10T07:26:17,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577106 2024-12-10T07:26:17,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577106) roll requested 2024-12-10T07:26:17,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577114 2024-12-10T07:26:17,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577114 2024-12-10T07:26:17,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577114) roll requested 2024-12-10T07:26:17,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577121 2024-12-10T07:26:17,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577121 2024-12-10T07:26:17,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577121) roll requested 2024-12-10T07:26:17,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577129 2024-12-10T07:26:17,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577129 2024-12-10T07:26:17,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577129) roll requested 2024-12-10T07:26:17,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577137 2024-12-10T07:26:17,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577137 2024-12-10T07:26:17,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577137) roll requested 2024-12-10T07:26:17,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577144 2024-12-10T07:26:17,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577144 2024-12-10T07:26:17,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577144) roll requested 2024-12-10T07:26:17,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577152 2024-12-10T07:26:17,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577152 2024-12-10T07:26:17,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577152) roll requested 2024-12-10T07:26:17,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577159 2024-12-10T07:26:17,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577159 2024-12-10T07:26:17,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577159) roll requested 2024-12-10T07:26:17,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577168 2024-12-10T07:26:17,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577168 2024-12-10T07:26:17,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577168) roll requested 2024-12-10T07:26:17,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577174 2024-12-10T07:26:17,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577174 2024-12-10T07:26:17,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577174) roll requested 2024-12-10T07:26:17,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577183 2024-12-10T07:26:17,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577183 2024-12-10T07:26:17,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577183) roll requested 2024-12-10T07:26:17,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577189 2024-12-10T07:26:17,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577189 2024-12-10T07:26:17,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577189) roll requested 2024-12-10T07:26:17,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577197 2024-12-10T07:26:17,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577197 2024-12-10T07:26:17,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577197) roll requested 2024-12-10T07:26:17,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577207 2024-12-10T07:26:17,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577207 2024-12-10T07:26:17,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577207) roll requested 2024-12-10T07:26:17,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577216 2024-12-10T07:26:17,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577216 2024-12-10T07:26:17,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577216) roll requested 2024-12-10T07:26:17,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577224 2024-12-10T07:26:17,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577224 2024-12-10T07:26:17,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577224) roll requested 2024-12-10T07:26:17,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577231 2024-12-10T07:26:17,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577231 2024-12-10T07:26:17,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577231) roll requested 2024-12-10T07:26:17,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577238 2024-12-10T07:26:17,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577238 2024-12-10T07:26:17,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577238) roll requested 2024-12-10T07:26:17,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577245 2024-12-10T07:26:17,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577245 2024-12-10T07:26:17,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577245) roll requested 2024-12-10T07:26:17,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577253 2024-12-10T07:26:17,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577253 2024-12-10T07:26:17,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577253) roll requested 2024-12-10T07:26:17,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577261 2024-12-10T07:26:17,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577261 2024-12-10T07:26:17,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577261) roll requested 2024-12-10T07:26:17,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577271 2024-12-10T07:26:17,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577271 2024-12-10T07:26:17,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577271) roll requested 2024-12-10T07:26:17,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577278 2024-12-10T07:26:17,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577278 2024-12-10T07:26:17,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577278) roll requested 2024-12-10T07:26:17,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577292 2024-12-10T07:26:17,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577292 2024-12-10T07:26:17,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577292) roll requested 2024-12-10T07:26:17,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577301 2024-12-10T07:26:17,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577301 2024-12-10T07:26:17,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577301) roll requested 2024-12-10T07:26:17,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577309 2024-12-10T07:26:17,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577309 2024-12-10T07:26:17,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577309) roll requested 2024-12-10T07:26:17,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577316 2024-12-10T07:26:17,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577316 2024-12-10T07:26:17,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577316) roll requested 2024-12-10T07:26:17,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577323 2024-12-10T07:26:17,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577323 2024-12-10T07:26:17,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577323) roll requested 2024-12-10T07:26:17,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577329 2024-12-10T07:26:17,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577329 2024-12-10T07:26:17,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577329) roll requested 2024-12-10T07:26:17,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577335 2024-12-10T07:26:17,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577335 2024-12-10T07:26:17,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577335) roll requested 2024-12-10T07:26:17,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577341 2024-12-10T07:26:17,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577341 2024-12-10T07:26:17,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577341) roll requested 2024-12-10T07:26:17,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577349 2024-12-10T07:26:17,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577349 2024-12-10T07:26:17,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577349) roll requested 2024-12-10T07:26:17,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577356 2024-12-10T07:26:17,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577356 2024-12-10T07:26:17,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577356) roll requested 2024-12-10T07:26:17,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577362 2024-12-10T07:26:17,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577362 2024-12-10T07:26:17,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577362) roll requested 2024-12-10T07:26:17,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577369 2024-12-10T07:26:17,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577369 2024-12-10T07:26:17,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577369) roll requested 2024-12-10T07:26:17,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577375 2024-12-10T07:26:17,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577375 2024-12-10T07:26:17,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577375) roll requested 2024-12-10T07:26:17,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577382 2024-12-10T07:26:17,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577382 2024-12-10T07:26:17,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577382) roll requested 2024-12-10T07:26:17,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577389 2024-12-10T07:26:17,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577389 2024-12-10T07:26:17,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577389) roll requested 2024-12-10T07:26:17,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577398 2024-12-10T07:26:17,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577398 2024-12-10T07:26:17,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577398) roll requested 2024-12-10T07:26:17,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577406 2024-12-10T07:26:17,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577406 2024-12-10T07:26:17,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577406) roll requested 2024-12-10T07:26:17,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577413 2024-12-10T07:26:17,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577413 2024-12-10T07:26:17,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577413) roll requested 2024-12-10T07:26:17,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577420 2024-12-10T07:26:17,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577420 2024-12-10T07:26:17,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577420) roll requested 2024-12-10T07:26:17,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577427 2024-12-10T07:26:17,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577427 2024-12-10T07:26:17,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577427) roll requested 2024-12-10T07:26:17,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577434 2024-12-10T07:26:17,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577434 2024-12-10T07:26:17,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577434) roll requested 2024-12-10T07:26:17,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577442 2024-12-10T07:26:17,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577442 2024-12-10T07:26:17,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577442) roll requested 2024-12-10T07:26:17,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577449 2024-12-10T07:26:17,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577449 2024-12-10T07:26:17,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577449) roll requested 2024-12-10T07:26:17,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577456 2024-12-10T07:26:17,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577456 2024-12-10T07:26:17,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577456) roll requested 2024-12-10T07:26:17,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577462 2024-12-10T07:26:17,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577462 2024-12-10T07:26:17,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577462) roll requested 2024-12-10T07:26:17,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577469 2024-12-10T07:26:17,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577469 2024-12-10T07:26:17,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577469) roll requested 2024-12-10T07:26:17,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577476 2024-12-10T07:26:17,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577476 2024-12-10T07:26:17,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577476) roll requested 2024-12-10T07:26:17,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577483 2024-12-10T07:26:17,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577483 2024-12-10T07:26:17,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577483) roll requested 2024-12-10T07:26:17,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577489 2024-12-10T07:26:17,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577489 2024-12-10T07:26:17,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577489) roll requested 2024-12-10T07:26:17,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577498 2024-12-10T07:26:17,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577498 2024-12-10T07:26:17,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577498) roll requested 2024-12-10T07:26:17,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577504 2024-12-10T07:26:17,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577504 2024-12-10T07:26:17,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577504) roll requested 2024-12-10T07:26:17,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577511 2024-12-10T07:26:17,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577511 2024-12-10T07:26:17,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577511) roll requested 2024-12-10T07:26:17,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577519 2024-12-10T07:26:17,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577519 2024-12-10T07:26:17,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577519) roll requested 2024-12-10T07:26:17,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577527 2024-12-10T07:26:17,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577527 2024-12-10T07:26:17,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577527) roll requested 2024-12-10T07:26:17,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577538 2024-12-10T07:26:17,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577538 2024-12-10T07:26:17,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577538) roll requested 2024-12-10T07:26:17,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577545 2024-12-10T07:26:17,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577545 2024-12-10T07:26:17,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577545) roll requested 2024-12-10T07:26:17,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577552 2024-12-10T07:26:17,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577552 2024-12-10T07:26:17,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577552) roll requested 2024-12-10T07:26:17,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577559 2024-12-10T07:26:17,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577559 2024-12-10T07:26:17,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577559) roll requested 2024-12-10T07:26:17,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577570 2024-12-10T07:26:17,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577570 2024-12-10T07:26:17,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577570) roll requested 2024-12-10T07:26:17,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577577 2024-12-10T07:26:17,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577577 2024-12-10T07:26:17,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577577) roll requested 2024-12-10T07:26:17,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577583 2024-12-10T07:26:17,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577583 2024-12-10T07:26:17,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577583) roll requested 2024-12-10T07:26:17,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577589 2024-12-10T07:26:17,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577589 2024-12-10T07:26:17,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577589) roll requested 2024-12-10T07:26:17,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577595 2024-12-10T07:26:17,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577595 2024-12-10T07:26:17,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577595) roll requested 2024-12-10T07:26:17,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577602 2024-12-10T07:26:17,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577602 2024-12-10T07:26:17,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577602) roll requested 2024-12-10T07:26:17,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577612 2024-12-10T07:26:17,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577612 2024-12-10T07:26:17,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577612) roll requested 2024-12-10T07:26:17,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577621 2024-12-10T07:26:17,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577621 2024-12-10T07:26:17,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577621) roll requested 2024-12-10T07:26:17,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577635 2024-12-10T07:26:17,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577635 2024-12-10T07:26:17,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577635) roll requested 2024-12-10T07:26:17,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577641 2024-12-10T07:26:17,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577641 2024-12-10T07:26:17,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577641) roll requested 2024-12-10T07:26:17,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577648 2024-12-10T07:26:17,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577648 2024-12-10T07:26:17,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577648) roll requested 2024-12-10T07:26:17,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577657 2024-12-10T07:26:17,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577657 2024-12-10T07:26:17,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577657) roll requested 2024-12-10T07:26:17,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577667 2024-12-10T07:26:17,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577667 2024-12-10T07:26:17,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577667) roll requested 2024-12-10T07:26:17,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577673 2024-12-10T07:26:17,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577673 2024-12-10T07:26:17,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577673) roll requested 2024-12-10T07:26:17,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577679 2024-12-10T07:26:17,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577679 2024-12-10T07:26:17,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577679) roll requested 2024-12-10T07:26:17,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577686 2024-12-10T07:26:17,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577686 2024-12-10T07:26:17,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577686) roll requested 2024-12-10T07:26:17,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577694 2024-12-10T07:26:17,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577694 2024-12-10T07:26:17,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577694) roll requested 2024-12-10T07:26:17,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577702 2024-12-10T07:26:17,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577702 2024-12-10T07:26:17,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577702) roll requested 2024-12-10T07:26:17,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577708 2024-12-10T07:26:17,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577708 2024-12-10T07:26:17,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577708) roll requested 2024-12-10T07:26:17,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577716 2024-12-10T07:26:17,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577716 2024-12-10T07:26:17,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577716) roll requested 2024-12-10T07:26:17,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577723 2024-12-10T07:26:17,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577723 2024-12-10T07:26:17,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577723) roll requested 2024-12-10T07:26:17,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577730 2024-12-10T07:26:17,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577730 2024-12-10T07:26:17,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577730) roll requested 2024-12-10T07:26:17,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577741 2024-12-10T07:26:17,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577741 2024-12-10T07:26:17,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577741) roll requested 2024-12-10T07:26:17,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577747 2024-12-10T07:26:17,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577747 2024-12-10T07:26:17,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577747) roll requested 2024-12-10T07:26:17,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577759 2024-12-10T07:26:17,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577759 2024-12-10T07:26:17,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577759) roll requested 2024-12-10T07:26:17,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577770 2024-12-10T07:26:17,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577770 2024-12-10T07:26:17,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577770) roll requested 2024-12-10T07:26:17,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577777 2024-12-10T07:26:17,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577777 2024-12-10T07:26:17,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577777) roll requested 2024-12-10T07:26:17,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577786 2024-12-10T07:26:17,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577786 2024-12-10T07:26:17,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577786) roll requested 2024-12-10T07:26:17,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577792 2024-12-10T07:26:17,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577792 2024-12-10T07:26:17,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577792) roll requested 2024-12-10T07:26:17,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577799 2024-12-10T07:26:17,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577799 2024-12-10T07:26:17,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577799) roll requested 2024-12-10T07:26:17,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577805 2024-12-10T07:26:17,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577805 2024-12-10T07:26:17,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577805) roll requested 2024-12-10T07:26:17,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577810 2024-12-10T07:26:17,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577810 2024-12-10T07:26:17,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577810) roll requested 2024-12-10T07:26:17,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577816 2024-12-10T07:26:17,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577816 2024-12-10T07:26:17,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577816) roll requested 2024-12-10T07:26:17,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577822 2024-12-10T07:26:17,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577822 2024-12-10T07:26:17,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577822) roll requested 2024-12-10T07:26:17,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577832 2024-12-10T07:26:17,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577832 2024-12-10T07:26:17,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577832) roll requested 2024-12-10T07:26:17,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577840 2024-12-10T07:26:17,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577840 2024-12-10T07:26:17,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577840) roll requested 2024-12-10T07:26:17,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577848 2024-12-10T07:26:17,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577848 2024-12-10T07:26:17,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577848) roll requested 2024-12-10T07:26:17,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577854 2024-12-10T07:26:17,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577854 2024-12-10T07:26:17,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577854) roll requested 2024-12-10T07:26:17,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577861 2024-12-10T07:26:17,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577861 2024-12-10T07:26:17,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577861) roll requested 2024-12-10T07:26:17,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577866 2024-12-10T07:26:17,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577866 2024-12-10T07:26:17,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577866) roll requested 2024-12-10T07:26:17,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577872 2024-12-10T07:26:17,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577872 2024-12-10T07:26:17,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577872) roll requested 2024-12-10T07:26:17,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577878 2024-12-10T07:26:17,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577878 2024-12-10T07:26:17,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577878) roll requested 2024-12-10T07:26:17,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577885 2024-12-10T07:26:17,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577885 2024-12-10T07:26:17,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577885) roll requested 2024-12-10T07:26:17,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577893 2024-12-10T07:26:17,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577893 2024-12-10T07:26:17,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577893) roll requested 2024-12-10T07:26:17,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577902 2024-12-10T07:26:17,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577902 2024-12-10T07:26:17,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577902) roll requested 2024-12-10T07:26:17,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577911 2024-12-10T07:26:17,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577911 2024-12-10T07:26:17,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577911) roll requested 2024-12-10T07:26:17,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577920 2024-12-10T07:26:17,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577920 2024-12-10T07:26:17,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577920) roll requested 2024-12-10T07:26:17,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577927 2024-12-10T07:26:17,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577927 2024-12-10T07:26:17,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577927) roll requested 2024-12-10T07:26:17,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577933 2024-12-10T07:26:17,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577933 2024-12-10T07:26:17,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577933) roll requested 2024-12-10T07:26:17,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577942 2024-12-10T07:26:17,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577942 2024-12-10T07:26:17,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577942) roll requested 2024-12-10T07:26:17,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577950 2024-12-10T07:26:17,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577950 2024-12-10T07:26:17,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577950) roll requested 2024-12-10T07:26:17,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577960 2024-12-10T07:26:17,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577960 2024-12-10T07:26:17,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577960) roll requested 2024-12-10T07:26:17,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577974 2024-12-10T07:26:17,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577974 2024-12-10T07:26:17,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577974) roll requested 2024-12-10T07:26:17,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577985 2024-12-10T07:26:17,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:17,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577985 2024-12-10T07:26:17,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:17,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:17,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577985) roll requested 2024-12-10T07:26:17,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815577993 2024-12-10T07:26:18,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577993 2024-12-10T07:26:18,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815577993) roll requested 2024-12-10T07:26:18,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578009 2024-12-10T07:26:18,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815577993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578009 2024-12-10T07:26:18,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578009) roll requested 2024-12-10T07:26:18,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578017 2024-12-10T07:26:18,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578017 2024-12-10T07:26:18,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578017) roll requested 2024-12-10T07:26:18,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578025 2024-12-10T07:26:18,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578025 2024-12-10T07:26:18,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578025) roll requested 2024-12-10T07:26:18,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578035 2024-12-10T07:26:18,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578035 2024-12-10T07:26:18,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578035) roll requested 2024-12-10T07:26:18,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578041 2024-12-10T07:26:18,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578041 2024-12-10T07:26:18,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578041) roll requested 2024-12-10T07:26:18,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578047 2024-12-10T07:26:18,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578047 2024-12-10T07:26:18,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578047) roll requested 2024-12-10T07:26:18,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578054 2024-12-10T07:26:18,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578054 2024-12-10T07:26:18,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578054) roll requested 2024-12-10T07:26:18,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578062 2024-12-10T07:26:18,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578062 2024-12-10T07:26:18,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578062) roll requested 2024-12-10T07:26:18,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578071 2024-12-10T07:26:18,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578071 2024-12-10T07:26:18,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578071) roll requested 2024-12-10T07:26:18,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578078 2024-12-10T07:26:18,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578078 2024-12-10T07:26:18,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578078) roll requested 2024-12-10T07:26:18,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578086 2024-12-10T07:26:18,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578086 2024-12-10T07:26:18,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578086) roll requested 2024-12-10T07:26:18,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578094 2024-12-10T07:26:18,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578094 2024-12-10T07:26:18,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578094) roll requested 2024-12-10T07:26:18,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578100 2024-12-10T07:26:18,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578100 2024-12-10T07:26:18,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578100) roll requested 2024-12-10T07:26:18,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578106 2024-12-10T07:26:18,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578106 2024-12-10T07:26:18,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578106) roll requested 2024-12-10T07:26:18,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578113 2024-12-10T07:26:18,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578113 2024-12-10T07:26:18,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578113) roll requested 2024-12-10T07:26:18,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578120 2024-12-10T07:26:18,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578120 2024-12-10T07:26:18,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578120) roll requested 2024-12-10T07:26:18,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578129 2024-12-10T07:26:18,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578129 2024-12-10T07:26:18,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578129) roll requested 2024-12-10T07:26:18,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578141 2024-12-10T07:26:18,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578141 2024-12-10T07:26:18,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578141) roll requested 2024-12-10T07:26:18,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578149 2024-12-10T07:26:18,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578149 2024-12-10T07:26:18,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578149) roll requested 2024-12-10T07:26:18,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578157 2024-12-10T07:26:18,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578157 2024-12-10T07:26:18,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578157) roll requested 2024-12-10T07:26:18,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578166 2024-12-10T07:26:18,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578166 2024-12-10T07:26:18,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578166) roll requested 2024-12-10T07:26:18,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578172 2024-12-10T07:26:18,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578172 2024-12-10T07:26:18,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578172) roll requested 2024-12-10T07:26:18,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578177 2024-12-10T07:26:18,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578177 2024-12-10T07:26:18,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578177) roll requested 2024-12-10T07:26:18,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578183 2024-12-10T07:26:18,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578183 2024-12-10T07:26:18,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578183) roll requested 2024-12-10T07:26:18,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578189 2024-12-10T07:26:18,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578189 2024-12-10T07:26:18,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578189) roll requested 2024-12-10T07:26:18,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578200 2024-12-10T07:26:18,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578200 2024-12-10T07:26:18,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578200) roll requested 2024-12-10T07:26:18,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578209 2024-12-10T07:26:18,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578209 2024-12-10T07:26:18,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578209) roll requested 2024-12-10T07:26:18,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578214 2024-12-10T07:26:18,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578214 2024-12-10T07:26:18,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578214) roll requested 2024-12-10T07:26:18,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578221 2024-12-10T07:26:18,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578221 2024-12-10T07:26:18,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578221) roll requested 2024-12-10T07:26:18,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578227 2024-12-10T07:26:18,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578227 2024-12-10T07:26:18,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578227) roll requested 2024-12-10T07:26:18,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578233 2024-12-10T07:26:18,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578233 2024-12-10T07:26:18,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578233) roll requested 2024-12-10T07:26:18,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578242 2024-12-10T07:26:18,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578242 2024-12-10T07:26:18,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578242) roll requested 2024-12-10T07:26:18,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578248 2024-12-10T07:26:18,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578248 2024-12-10T07:26:18,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578248) roll requested 2024-12-10T07:26:18,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578256 2024-12-10T07:26:18,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578256 2024-12-10T07:26:18,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578256) roll requested 2024-12-10T07:26:18,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578263 2024-12-10T07:26:18,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578263 2024-12-10T07:26:18,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578263) roll requested 2024-12-10T07:26:18,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578270 2024-12-10T07:26:18,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578270 2024-12-10T07:26:18,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578270) roll requested 2024-12-10T07:26:18,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578276 2024-12-10T07:26:18,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578276 2024-12-10T07:26:18,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578276) roll requested 2024-12-10T07:26:18,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578284 2024-12-10T07:26:18,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578284 2024-12-10T07:26:18,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578284) roll requested 2024-12-10T07:26:18,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578292 2024-12-10T07:26:18,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578292 2024-12-10T07:26:18,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578292) roll requested 2024-12-10T07:26:18,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578302 2024-12-10T07:26:18,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578302 2024-12-10T07:26:18,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578302) roll requested 2024-12-10T07:26:18,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578310 2024-12-10T07:26:18,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578310 2024-12-10T07:26:18,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578310) roll requested 2024-12-10T07:26:18,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578316 2024-12-10T07:26:18,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578316 2024-12-10T07:26:18,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578316) roll requested 2024-12-10T07:26:18,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578322 2024-12-10T07:26:18,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578322 2024-12-10T07:26:18,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578322) roll requested 2024-12-10T07:26:18,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578327 2024-12-10T07:26:18,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578327 2024-12-10T07:26:18,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578327) roll requested 2024-12-10T07:26:18,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578336 2024-12-10T07:26:18,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578336 2024-12-10T07:26:18,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578336) roll requested 2024-12-10T07:26:18,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578345 2024-12-10T07:26:18,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578345 2024-12-10T07:26:18,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578345) roll requested 2024-12-10T07:26:18,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578354 2024-12-10T07:26:18,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578354 2024-12-10T07:26:18,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578354) roll requested 2024-12-10T07:26:18,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578364 2024-12-10T07:26:18,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578364 2024-12-10T07:26:18,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578364) roll requested 2024-12-10T07:26:18,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578371 2024-12-10T07:26:18,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578371 2024-12-10T07:26:18,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578371) roll requested 2024-12-10T07:26:18,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578378 2024-12-10T07:26:18,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578378 2024-12-10T07:26:18,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578378) roll requested 2024-12-10T07:26:18,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578387 2024-12-10T07:26:18,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578387 2024-12-10T07:26:18,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578387) roll requested 2024-12-10T07:26:18,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578396 2024-12-10T07:26:18,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578396 2024-12-10T07:26:18,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578396) roll requested 2024-12-10T07:26:18,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578404 2024-12-10T07:26:18,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578404 2024-12-10T07:26:18,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578404) roll requested 2024-12-10T07:26:18,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578412 2024-12-10T07:26:18,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578412 2024-12-10T07:26:18,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578412) roll requested 2024-12-10T07:26:18,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578421 2024-12-10T07:26:18,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578421 2024-12-10T07:26:18,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578421) roll requested 2024-12-10T07:26:18,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578429 2024-12-10T07:26:18,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578429 2024-12-10T07:26:18,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578429) roll requested 2024-12-10T07:26:18,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578438 2024-12-10T07:26:18,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578438 2024-12-10T07:26:18,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578438) roll requested 2024-12-10T07:26:18,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578446 2024-12-10T07:26:18,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578446 2024-12-10T07:26:18,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578446) roll requested 2024-12-10T07:26:18,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578451 2024-12-10T07:26:18,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578451 2024-12-10T07:26:18,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578451) roll requested 2024-12-10T07:26:18,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578457 2024-12-10T07:26:18,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578457 2024-12-10T07:26:18,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578457) roll requested 2024-12-10T07:26:18,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578465 2024-12-10T07:26:18,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578465 2024-12-10T07:26:18,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578465) roll requested 2024-12-10T07:26:18,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578471 2024-12-10T07:26:18,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578471 2024-12-10T07:26:18,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578471) roll requested 2024-12-10T07:26:18,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578480 2024-12-10T07:26:18,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578480 2024-12-10T07:26:18,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578480) roll requested 2024-12-10T07:26:18,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578486 2024-12-10T07:26:18,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578486 2024-12-10T07:26:18,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578486) roll requested 2024-12-10T07:26:18,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578492 2024-12-10T07:26:18,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578492 2024-12-10T07:26:18,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578492) roll requested 2024-12-10T07:26:18,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578499 2024-12-10T07:26:18,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578499 2024-12-10T07:26:18,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578499) roll requested 2024-12-10T07:26:18,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578505 2024-12-10T07:26:18,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578505 2024-12-10T07:26:18,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578505) roll requested 2024-12-10T07:26:18,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578514 2024-12-10T07:26:18,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578514 2024-12-10T07:26:18,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578514) roll requested 2024-12-10T07:26:18,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578522 2024-12-10T07:26:18,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578522 2024-12-10T07:26:18,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578522) roll requested 2024-12-10T07:26:18,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578535 2024-12-10T07:26:18,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578535 2024-12-10T07:26:18,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578535) roll requested 2024-12-10T07:26:18,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578541 2024-12-10T07:26:18,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578541 2024-12-10T07:26:18,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578541) roll requested 2024-12-10T07:26:18,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578547 2024-12-10T07:26:18,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578547 2024-12-10T07:26:18,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578547) roll requested 2024-12-10T07:26:18,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578555 2024-12-10T07:26:18,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578555 2024-12-10T07:26:18,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578555) roll requested 2024-12-10T07:26:18,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578562 2024-12-10T07:26:18,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578562 2024-12-10T07:26:18,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578562) roll requested 2024-12-10T07:26:18,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578571 2024-12-10T07:26:18,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578571 2024-12-10T07:26:18,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578571) roll requested 2024-12-10T07:26:18,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578579 2024-12-10T07:26:18,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578579 2024-12-10T07:26:18,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578579) roll requested 2024-12-10T07:26:18,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578589 2024-12-10T07:26:18,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578589 2024-12-10T07:26:18,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578589) roll requested 2024-12-10T07:26:18,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578597 2024-12-10T07:26:18,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578597 2024-12-10T07:26:18,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578597) roll requested 2024-12-10T07:26:18,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578604 2024-12-10T07:26:18,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578604 2024-12-10T07:26:18,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578604) roll requested 2024-12-10T07:26:18,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578616 2024-12-10T07:26:18,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578616 2024-12-10T07:26:18,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578616) roll requested 2024-12-10T07:26:18,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578625 2024-12-10T07:26:18,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578625 2024-12-10T07:26:18,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578625) roll requested 2024-12-10T07:26:18,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578633 2024-12-10T07:26:18,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578633 2024-12-10T07:26:18,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578633) roll requested 2024-12-10T07:26:18,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578639 2024-12-10T07:26:18,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578639 2024-12-10T07:26:18,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578639) roll requested 2024-12-10T07:26:18,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578644 2024-12-10T07:26:18,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578644 2024-12-10T07:26:18,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578644) roll requested 2024-12-10T07:26:18,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578650 2024-12-10T07:26:18,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578650 2024-12-10T07:26:18,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578650) roll requested 2024-12-10T07:26:18,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578657 2024-12-10T07:26:18,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578657 2024-12-10T07:26:18,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578657) roll requested 2024-12-10T07:26:18,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578665 2024-12-10T07:26:18,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578665 2024-12-10T07:26:18,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578665) roll requested 2024-12-10T07:26:18,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578674 2024-12-10T07:26:18,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578674 2024-12-10T07:26:18,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578674) roll requested 2024-12-10T07:26:18,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578682 2024-12-10T07:26:18,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578682 2024-12-10T07:26:18,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578682) roll requested 2024-12-10T07:26:18,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578689 2024-12-10T07:26:18,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578689 2024-12-10T07:26:18,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578689) roll requested 2024-12-10T07:26:18,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578695 2024-12-10T07:26:18,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578695 2024-12-10T07:26:18,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578695) roll requested 2024-12-10T07:26:18,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578701 2024-12-10T07:26:18,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578701 2024-12-10T07:26:18,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578701) roll requested 2024-12-10T07:26:18,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578707 2024-12-10T07:26:18,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578707 2024-12-10T07:26:18,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578707) roll requested 2024-12-10T07:26:18,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578719 2024-12-10T07:26:18,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578719 2024-12-10T07:26:18,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578719) roll requested 2024-12-10T07:26:18,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578725 2024-12-10T07:26:18,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578725 2024-12-10T07:26:18,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578725) roll requested 2024-12-10T07:26:18,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578736 2024-12-10T07:26:18,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578736 2024-12-10T07:26:18,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578736) roll requested 2024-12-10T07:26:18,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578742 2024-12-10T07:26:18,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578742 2024-12-10T07:26:18,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578742) roll requested 2024-12-10T07:26:18,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578749 2024-12-10T07:26:18,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578749 2024-12-10T07:26:18,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578749) roll requested 2024-12-10T07:26:18,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578756 2024-12-10T07:26:18,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578756 2024-12-10T07:26:18,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578756) roll requested 2024-12-10T07:26:18,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578762 2024-12-10T07:26:18,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578762 2024-12-10T07:26:18,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578762) roll requested 2024-12-10T07:26:18,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578768 2024-12-10T07:26:18,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578768 2024-12-10T07:26:18,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578768) roll requested 2024-12-10T07:26:18,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578775 2024-12-10T07:26:18,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578775 2024-12-10T07:26:18,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578775) roll requested 2024-12-10T07:26:18,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578782 2024-12-10T07:26:18,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578782 2024-12-10T07:26:18,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578782) roll requested 2024-12-10T07:26:18,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578790 2024-12-10T07:26:18,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578790 2024-12-10T07:26:18,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578790) roll requested 2024-12-10T07:26:18,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578796 2024-12-10T07:26:18,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578796 2024-12-10T07:26:18,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578796) roll requested 2024-12-10T07:26:18,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578802 2024-12-10T07:26:18,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578802 2024-12-10T07:26:18,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578802) roll requested 2024-12-10T07:26:18,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578810 2024-12-10T07:26:18,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578810 2024-12-10T07:26:18,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578810) roll requested 2024-12-10T07:26:18,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578818 2024-12-10T07:26:18,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578818 2024-12-10T07:26:18,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578818) roll requested 2024-12-10T07:26:18,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578827 2024-12-10T07:26:18,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578827 2024-12-10T07:26:18,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578827) roll requested 2024-12-10T07:26:18,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578836 2024-12-10T07:26:18,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578836 2024-12-10T07:26:18,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578836) roll requested 2024-12-10T07:26:18,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578845 2024-12-10T07:26:18,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578845 2024-12-10T07:26:18,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578845) roll requested 2024-12-10T07:26:18,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578854 2024-12-10T07:26:18,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578854 2024-12-10T07:26:18,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578854) roll requested 2024-12-10T07:26:18,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578864 2024-12-10T07:26:18,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578864 2024-12-10T07:26:18,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578864) roll requested 2024-12-10T07:26:18,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578873 2024-12-10T07:26:18,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578873 2024-12-10T07:26:18,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578873) roll requested 2024-12-10T07:26:18,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578881 2024-12-10T07:26:18,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578881 2024-12-10T07:26:18,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578881) roll requested 2024-12-10T07:26:18,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578888 2024-12-10T07:26:18,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578888 2024-12-10T07:26:18,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578888) roll requested 2024-12-10T07:26:18,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578897 2024-12-10T07:26:18,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578897 2024-12-10T07:26:18,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578897) roll requested 2024-12-10T07:26:18,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578906 2024-12-10T07:26:18,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578906 2024-12-10T07:26:18,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578906) roll requested 2024-12-10T07:26:18,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578915 2024-12-10T07:26:18,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578915 2024-12-10T07:26:18,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578915) roll requested 2024-12-10T07:26:18,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578920 2024-12-10T07:26:18,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578920 2024-12-10T07:26:18,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578920) roll requested 2024-12-10T07:26:18,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578926 2024-12-10T07:26:18,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578926 2024-12-10T07:26:18,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578926) roll requested 2024-12-10T07:26:18,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578933 2024-12-10T07:26:18,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578933 2024-12-10T07:26:18,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578933) roll requested 2024-12-10T07:26:18,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578940 2024-12-10T07:26:18,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578940 2024-12-10T07:26:18,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578940) roll requested 2024-12-10T07:26:18,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578948 2024-12-10T07:26:18,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578948 2024-12-10T07:26:18,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578948) roll requested 2024-12-10T07:26:18,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578959 2024-12-10T07:26:18,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578959 2024-12-10T07:26:18,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578959) roll requested 2024-12-10T07:26:18,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578973 2024-12-10T07:26:18,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578973 2024-12-10T07:26:18,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578973) roll requested 2024-12-10T07:26:18,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578981 2024-12-10T07:26:18,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578981 2024-12-10T07:26:18,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578981) roll requested 2024-12-10T07:26:18,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578986 2024-12-10T07:26:18,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578986 2024-12-10T07:26:18,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578986) roll requested 2024-12-10T07:26:18,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578992 2024-12-10T07:26:18,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:18,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578992 2024-12-10T07:26:18,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:18,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:18,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578992) roll requested 2024-12-10T07:26:18,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815578999 2024-12-10T07:26:19,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578999 2024-12-10T07:26:19,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815578999) roll requested 2024-12-10T07:26:19,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579007 2024-12-10T07:26:19,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815578999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579007 2024-12-10T07:26:19,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579007) roll requested 2024-12-10T07:26:19,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579016 2024-12-10T07:26:19,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579016 2024-12-10T07:26:19,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579016) roll requested 2024-12-10T07:26:19,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579025 2024-12-10T07:26:19,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579025 2024-12-10T07:26:19,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579025) roll requested 2024-12-10T07:26:19,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579033 2024-12-10T07:26:19,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579033 2024-12-10T07:26:19,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579033) roll requested 2024-12-10T07:26:19,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579041 2024-12-10T07:26:19,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579041 2024-12-10T07:26:19,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579041) roll requested 2024-12-10T07:26:19,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579048 2024-12-10T07:26:19,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579048 2024-12-10T07:26:19,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579048) roll requested 2024-12-10T07:26:19,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579055 2024-12-10T07:26:19,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579055 2024-12-10T07:26:19,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579055) roll requested 2024-12-10T07:26:19,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579063 2024-12-10T07:26:19,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579063 2024-12-10T07:26:19,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579063) roll requested 2024-12-10T07:26:19,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579072 2024-12-10T07:26:19,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579072 2024-12-10T07:26:19,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579072) roll requested 2024-12-10T07:26:19,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579082 2024-12-10T07:26:19,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579082 2024-12-10T07:26:19,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579082) roll requested 2024-12-10T07:26:19,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579091 2024-12-10T07:26:19,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579091 2024-12-10T07:26:19,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579091) roll requested 2024-12-10T07:26:19,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579099 2024-12-10T07:26:19,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579099 2024-12-10T07:26:19,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579099) roll requested 2024-12-10T07:26:19,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579108 2024-12-10T07:26:19,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579108 2024-12-10T07:26:19,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579108) roll requested 2024-12-10T07:26:19,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579116 2024-12-10T07:26:19,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579116 2024-12-10T07:26:19,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579116) roll requested 2024-12-10T07:26:19,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579126 2024-12-10T07:26:19,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579126 2024-12-10T07:26:19,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579126) roll requested 2024-12-10T07:26:19,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579134 2024-12-10T07:26:19,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579134 2024-12-10T07:26:19,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579134) roll requested 2024-12-10T07:26:19,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579139 2024-12-10T07:26:19,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579139 2024-12-10T07:26:19,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579139) roll requested 2024-12-10T07:26:19,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579145 2024-12-10T07:26:19,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579145 2024-12-10T07:26:19,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579145) roll requested 2024-12-10T07:26:19,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579152 2024-12-10T07:26:19,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579152 2024-12-10T07:26:19,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579152) roll requested 2024-12-10T07:26:19,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579159 2024-12-10T07:26:19,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579159 2024-12-10T07:26:19,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579159) roll requested 2024-12-10T07:26:19,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579170 2024-12-10T07:26:19,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579170 2024-12-10T07:26:19,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579170) roll requested 2024-12-10T07:26:19,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579177 2024-12-10T07:26:19,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579177 2024-12-10T07:26:19,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579177) roll requested 2024-12-10T07:26:19,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579184 2024-12-10T07:26:19,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579184 2024-12-10T07:26:19,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579184) roll requested 2024-12-10T07:26:19,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579193 2024-12-10T07:26:19,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579193 2024-12-10T07:26:19,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579193) roll requested 2024-12-10T07:26:19,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579202 2024-12-10T07:26:19,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579202 2024-12-10T07:26:19,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579202) roll requested 2024-12-10T07:26:19,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579212 2024-12-10T07:26:19,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579212 2024-12-10T07:26:19,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579212) roll requested 2024-12-10T07:26:19,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579221 2024-12-10T07:26:19,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579221 2024-12-10T07:26:19,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579221) roll requested 2024-12-10T07:26:19,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579229 2024-12-10T07:26:19,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579229 2024-12-10T07:26:19,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579229) roll requested 2024-12-10T07:26:19,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579240 2024-12-10T07:26:19,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579240 2024-12-10T07:26:19,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579240) roll requested 2024-12-10T07:26:19,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579246 2024-12-10T07:26:19,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579246 2024-12-10T07:26:19,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579246) roll requested 2024-12-10T07:26:19,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579253 2024-12-10T07:26:19,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579253 2024-12-10T07:26:19,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579253) roll requested 2024-12-10T07:26:19,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579262 2024-12-10T07:26:19,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579262 2024-12-10T07:26:19,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579262) roll requested 2024-12-10T07:26:19,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579269 2024-12-10T07:26:19,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579269 2024-12-10T07:26:19,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579269) roll requested 2024-12-10T07:26:19,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579278 2024-12-10T07:26:19,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579278 2024-12-10T07:26:19,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579278) roll requested 2024-12-10T07:26:19,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579288 2024-12-10T07:26:19,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579288 2024-12-10T07:26:19,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579288) roll requested 2024-12-10T07:26:19,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579298 2024-12-10T07:26:19,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579298 2024-12-10T07:26:19,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579298) roll requested 2024-12-10T07:26:19,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579306 2024-12-10T07:26:19,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579306 2024-12-10T07:26:19,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579306) roll requested 2024-12-10T07:26:19,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579314 2024-12-10T07:26:19,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579314 2024-12-10T07:26:19,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579314) roll requested 2024-12-10T07:26:19,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579321 2024-12-10T07:26:19,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579321 2024-12-10T07:26:19,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579321) roll requested 2024-12-10T07:26:19,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579329 2024-12-10T07:26:19,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579329 2024-12-10T07:26:19,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579329) roll requested 2024-12-10T07:26:19,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579339 2024-12-10T07:26:19,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579339 2024-12-10T07:26:19,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579339) roll requested 2024-12-10T07:26:19,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579346 2024-12-10T07:26:19,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579346 2024-12-10T07:26:19,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579346) roll requested 2024-12-10T07:26:19,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579351 2024-12-10T07:26:19,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579351 2024-12-10T07:26:19,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579351) roll requested 2024-12-10T07:26:19,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579365 2024-12-10T07:26:19,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579365 2024-12-10T07:26:19,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579365) roll requested 2024-12-10T07:26:19,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579377 2024-12-10T07:26:19,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579377 2024-12-10T07:26:19,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579377) roll requested 2024-12-10T07:26:19,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579382 2024-12-10T07:26:19,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579382 2024-12-10T07:26:19,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579382) roll requested 2024-12-10T07:26:19,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579387 2024-12-10T07:26:19,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579387 2024-12-10T07:26:19,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579387) roll requested 2024-12-10T07:26:19,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579393 2024-12-10T07:26:19,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579393 2024-12-10T07:26:19,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579393) roll requested 2024-12-10T07:26:19,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579399 2024-12-10T07:26:19,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579399 2024-12-10T07:26:19,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579399) roll requested 2024-12-10T07:26:19,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579407 2024-12-10T07:26:19,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579407 2024-12-10T07:26:19,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579407) roll requested 2024-12-10T07:26:19,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579416 2024-12-10T07:26:19,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579416 2024-12-10T07:26:19,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579416) roll requested 2024-12-10T07:26:19,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579425 2024-12-10T07:26:19,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579425 2024-12-10T07:26:19,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579425) roll requested 2024-12-10T07:26:19,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579436 2024-12-10T07:26:19,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579436 2024-12-10T07:26:19,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579436) roll requested 2024-12-10T07:26:19,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579446 2024-12-10T07:26:19,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579446 2024-12-10T07:26:19,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579446) roll requested 2024-12-10T07:26:19,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579455 2024-12-10T07:26:19,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579455 2024-12-10T07:26:19,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579455) roll requested 2024-12-10T07:26:19,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579461 2024-12-10T07:26:19,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579461 2024-12-10T07:26:19,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579461) roll requested 2024-12-10T07:26:19,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579466 2024-12-10T07:26:19,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579466 2024-12-10T07:26:19,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579466) roll requested 2024-12-10T07:26:19,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579472 2024-12-10T07:26:19,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579472 2024-12-10T07:26:19,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579472) roll requested 2024-12-10T07:26:19,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579477 2024-12-10T07:26:19,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579477 2024-12-10T07:26:19,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579477) roll requested 2024-12-10T07:26:19,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579485 2024-12-10T07:26:19,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579485 2024-12-10T07:26:19,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579485) roll requested 2024-12-10T07:26:19,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579495 2024-12-10T07:26:19,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579495 2024-12-10T07:26:19,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579495) roll requested 2024-12-10T07:26:19,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579502 2024-12-10T07:26:19,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579502 2024-12-10T07:26:19,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579502) roll requested 2024-12-10T07:26:19,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579511 2024-12-10T07:26:19,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579511 2024-12-10T07:26:19,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579511) roll requested 2024-12-10T07:26:19,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579520 2024-12-10T07:26:19,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579520 2024-12-10T07:26:19,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579520) roll requested 2024-12-10T07:26:19,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579526 2024-12-10T07:26:19,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579526 2024-12-10T07:26:19,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579526) roll requested 2024-12-10T07:26:19,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579534 2024-12-10T07:26:19,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579534 2024-12-10T07:26:19,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579534) roll requested 2024-12-10T07:26:19,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579541 2024-12-10T07:26:19,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579541 2024-12-10T07:26:19,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579541) roll requested 2024-12-10T07:26:19,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579548 2024-12-10T07:26:19,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579548 2024-12-10T07:26:19,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579548) roll requested 2024-12-10T07:26:19,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579555 2024-12-10T07:26:19,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579555 2024-12-10T07:26:19,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579555) roll requested 2024-12-10T07:26:19,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579561 2024-12-10T07:26:19,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579561 2024-12-10T07:26:19,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579561) roll requested 2024-12-10T07:26:19,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579566 2024-12-10T07:26:19,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579566 2024-12-10T07:26:19,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579566) roll requested 2024-12-10T07:26:19,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579572 2024-12-10T07:26:19,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579572 2024-12-10T07:26:19,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579572) roll requested 2024-12-10T07:26:19,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579577 2024-12-10T07:26:19,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579577 2024-12-10T07:26:19,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579577) roll requested 2024-12-10T07:26:19,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579582 2024-12-10T07:26:19,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579582 2024-12-10T07:26:19,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579582) roll requested 2024-12-10T07:26:19,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579588 2024-12-10T07:26:19,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579588 2024-12-10T07:26:19,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579588) roll requested 2024-12-10T07:26:19,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579595 2024-12-10T07:26:19,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579595 2024-12-10T07:26:19,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579595) roll requested 2024-12-10T07:26:19,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579604 2024-12-10T07:26:19,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579604 2024-12-10T07:26:19,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579604) roll requested 2024-12-10T07:26:19,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579609 2024-12-10T07:26:19,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579609 2024-12-10T07:26:19,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579609) roll requested 2024-12-10T07:26:19,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579614 2024-12-10T07:26:19,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579614 2024-12-10T07:26:19,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579614) roll requested 2024-12-10T07:26:19,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579620 2024-12-10T07:26:19,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579620 2024-12-10T07:26:19,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579620) roll requested 2024-12-10T07:26:19,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579627 2024-12-10T07:26:19,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579627 2024-12-10T07:26:19,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579627) roll requested 2024-12-10T07:26:19,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579634 2024-12-10T07:26:19,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579634 2024-12-10T07:26:19,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579634) roll requested 2024-12-10T07:26:19,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579642 2024-12-10T07:26:19,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579642 2024-12-10T07:26:19,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579642) roll requested 2024-12-10T07:26:19,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579649 2024-12-10T07:26:19,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579649 2024-12-10T07:26:19,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579649) roll requested 2024-12-10T07:26:19,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579659 2024-12-10T07:26:19,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579659 2024-12-10T07:26:19,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579659) roll requested 2024-12-10T07:26:19,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579668 2024-12-10T07:26:19,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579668 2024-12-10T07:26:19,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579668) roll requested 2024-12-10T07:26:19,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579677 2024-12-10T07:26:19,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579677 2024-12-10T07:26:19,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579677) roll requested 2024-12-10T07:26:19,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579691 2024-12-10T07:26:19,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579691 2024-12-10T07:26:19,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579691) roll requested 2024-12-10T07:26:19,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579700 2024-12-10T07:26:19,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579700 2024-12-10T07:26:19,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579700) roll requested 2024-12-10T07:26:19,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579705 2024-12-10T07:26:19,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579705 2024-12-10T07:26:19,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579705) roll requested 2024-12-10T07:26:19,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579710 2024-12-10T07:26:19,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579710 2024-12-10T07:26:19,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579710) roll requested 2024-12-10T07:26:19,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579716 2024-12-10T07:26:19,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579716 2024-12-10T07:26:19,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579716) roll requested 2024-12-10T07:26:19,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579723 2024-12-10T07:26:19,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579723 2024-12-10T07:26:19,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579723) roll requested 2024-12-10T07:26:19,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579731 2024-12-10T07:26:19,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579731 2024-12-10T07:26:19,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579731) roll requested 2024-12-10T07:26:19,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579739 2024-12-10T07:26:19,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579739 2024-12-10T07:26:19,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579739) roll requested 2024-12-10T07:26:19,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579752 2024-12-10T07:26:19,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579752 2024-12-10T07:26:19,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579752) roll requested 2024-12-10T07:26:19,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579765 2024-12-10T07:26:19,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579765 2024-12-10T07:26:19,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579765) roll requested 2024-12-10T07:26:19,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579771 2024-12-10T07:26:19,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579771 2024-12-10T07:26:19,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579771) roll requested 2024-12-10T07:26:19,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579781 2024-12-10T07:26:19,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579781 2024-12-10T07:26:19,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579781) roll requested 2024-12-10T07:26:19,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579791 2024-12-10T07:26:19,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579791 2024-12-10T07:26:19,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579791) roll requested 2024-12-10T07:26:19,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579796 2024-12-10T07:26:19,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579796 2024-12-10T07:26:19,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579796) roll requested 2024-12-10T07:26:19,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579802 2024-12-10T07:26:19,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579802 2024-12-10T07:26:19,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579802) roll requested 2024-12-10T07:26:19,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579809 2024-12-10T07:26:19,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579809 2024-12-10T07:26:19,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579809) roll requested 2024-12-10T07:26:19,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579817 2024-12-10T07:26:19,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579817 2024-12-10T07:26:19,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579817) roll requested 2024-12-10T07:26:19,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579824 2024-12-10T07:26:19,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579824 2024-12-10T07:26:19,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579824) roll requested 2024-12-10T07:26:19,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579831 2024-12-10T07:26:19,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579831 2024-12-10T07:26:19,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579831) roll requested 2024-12-10T07:26:19,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579840 2024-12-10T07:26:19,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579840 2024-12-10T07:26:19,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579840) roll requested 2024-12-10T07:26:19,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579849 2024-12-10T07:26:19,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579849 2024-12-10T07:26:19,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579849) roll requested 2024-12-10T07:26:19,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579857 2024-12-10T07:26:19,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579857 2024-12-10T07:26:19,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579857) roll requested 2024-12-10T07:26:19,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579866 2024-12-10T07:26:19,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579866 2024-12-10T07:26:19,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579866) roll requested 2024-12-10T07:26:19,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579877 2024-12-10T07:26:19,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579877 2024-12-10T07:26:19,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579877) roll requested 2024-12-10T07:26:19,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579886 2024-12-10T07:26:19,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579886 2024-12-10T07:26:19,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579886) roll requested 2024-12-10T07:26:19,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579894 2024-12-10T07:26:19,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579894 2024-12-10T07:26:19,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579894) roll requested 2024-12-10T07:26:19,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579901 2024-12-10T07:26:19,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579901 2024-12-10T07:26:19,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579901) roll requested 2024-12-10T07:26:19,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579908 2024-12-10T07:26:19,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579908 2024-12-10T07:26:19,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579908) roll requested 2024-12-10T07:26:19,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579919 2024-12-10T07:26:19,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579919 2024-12-10T07:26:19,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579919) roll requested 2024-12-10T07:26:19,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579928 2024-12-10T07:26:19,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579928 2024-12-10T07:26:19,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579928) roll requested 2024-12-10T07:26:19,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579937 2024-12-10T07:26:19,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579937 2024-12-10T07:26:19,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579937) roll requested 2024-12-10T07:26:19,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579949 2024-12-10T07:26:19,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579949 2024-12-10T07:26:19,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579949) roll requested 2024-12-10T07:26:19,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579954 2024-12-10T07:26:19,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579954 2024-12-10T07:26:19,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579954) roll requested 2024-12-10T07:26:19,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579959 2024-12-10T07:26:19,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579959 2024-12-10T07:26:19,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579959) roll requested 2024-12-10T07:26:19,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579965 2024-12-10T07:26:19,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579965 2024-12-10T07:26:19,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579965) roll requested 2024-12-10T07:26:19,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579972 2024-12-10T07:26:19,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579972 2024-12-10T07:26:19,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579972) roll requested 2024-12-10T07:26:19,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579980 2024-12-10T07:26:19,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579980 2024-12-10T07:26:19,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579980) roll requested 2024-12-10T07:26:19,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579988 2024-12-10T07:26:19,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:19,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579988 2024-12-10T07:26:19,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:19,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:19,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579988) roll requested 2024-12-10T07:26:19,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815579997 2024-12-10T07:26:20,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579997 2024-12-10T07:26:20,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815579997) roll requested 2024-12-10T07:26:20,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580007 2024-12-10T07:26:20,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815579997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580007 2024-12-10T07:26:20,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580007) roll requested 2024-12-10T07:26:20,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580018 2024-12-10T07:26:20,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580018 2024-12-10T07:26:20,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580018) roll requested 2024-12-10T07:26:20,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580026 2024-12-10T07:26:20,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580026 2024-12-10T07:26:20,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580026) roll requested 2024-12-10T07:26:20,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580034 2024-12-10T07:26:20,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580034 2024-12-10T07:26:20,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580034) roll requested 2024-12-10T07:26:20,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580042 2024-12-10T07:26:20,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580042 2024-12-10T07:26:20,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580042) roll requested 2024-12-10T07:26:20,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580048 2024-12-10T07:26:20,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580048 2024-12-10T07:26:20,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580048) roll requested 2024-12-10T07:26:20,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580054 2024-12-10T07:26:20,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580054 2024-12-10T07:26:20,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580054) roll requested 2024-12-10T07:26:20,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580066 2024-12-10T07:26:20,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580066 2024-12-10T07:26:20,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580066) roll requested 2024-12-10T07:26:20,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580080 2024-12-10T07:26:20,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580080 2024-12-10T07:26:20,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580080) roll requested 2024-12-10T07:26:20,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580086 2024-12-10T07:26:20,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580086 2024-12-10T07:26:20,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580080 is not closed yet, will try archiving it next time 2024-12-10T07:26:20,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580086) roll requested 2024-12-10T07:26:20,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580091 2024-12-10T07:26:20,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580091 2024-12-10T07:26:20,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580091) roll requested 2024-12-10T07:26:20,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580145 2024-12-10T07:26:20,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580145 2024-12-10T07:26:20,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580145) roll requested 2024-12-10T07:26:20,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580153 2024-12-10T07:26:20,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580153 2024-12-10T07:26:20,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580153) roll requested 2024-12-10T07:26:20,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580160 2024-12-10T07:26:20,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580160 2024-12-10T07:26:20,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580160) roll requested 2024-12-10T07:26:20,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580167 2024-12-10T07:26:20,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580167 2024-12-10T07:26:20,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580167) roll requested 2024-12-10T07:26:20,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580174 2024-12-10T07:26:20,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580174 2024-12-10T07:26:20,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580174) roll requested 2024-12-10T07:26:20,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580184 2024-12-10T07:26:20,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580184 2024-12-10T07:26:20,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580184) roll requested 2024-12-10T07:26:20,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580193 2024-12-10T07:26:20,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580193 2024-12-10T07:26:20,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580193) roll requested 2024-12-10T07:26:20,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580202 2024-12-10T07:26:20,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580202 2024-12-10T07:26:20,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580202) roll requested 2024-12-10T07:26:20,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580212 2024-12-10T07:26:20,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580212 2024-12-10T07:26:20,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580212) roll requested 2024-12-10T07:26:20,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580223 2024-12-10T07:26:20,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580223 2024-12-10T07:26:20,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580223) roll requested 2024-12-10T07:26:20,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580229 2024-12-10T07:26:20,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580229 2024-12-10T07:26:20,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580229) roll requested 2024-12-10T07:26:20,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580235 2024-12-10T07:26:20,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580235 2024-12-10T07:26:20,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580235) roll requested 2024-12-10T07:26:20,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580241 2024-12-10T07:26:20,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580241 2024-12-10T07:26:20,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580241) roll requested 2024-12-10T07:26:20,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580247 2024-12-10T07:26:20,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580247 2024-12-10T07:26:20,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580247) roll requested 2024-12-10T07:26:20,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580255 2024-12-10T07:26:20,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580255 2024-12-10T07:26:20,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580255) roll requested 2024-12-10T07:26:20,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580264 2024-12-10T07:26:20,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580264 2024-12-10T07:26:20,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580264) roll requested 2024-12-10T07:26:20,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580271 2024-12-10T07:26:20,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580271 2024-12-10T07:26:20,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580271) roll requested 2024-12-10T07:26:20,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580277 2024-12-10T07:26:20,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580277 2024-12-10T07:26:20,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580277) roll requested 2024-12-10T07:26:20,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580284 2024-12-10T07:26:20,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580284 2024-12-10T07:26:20,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580284) roll requested 2024-12-10T07:26:20,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580292 2024-12-10T07:26:20,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580292 2024-12-10T07:26:20,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580292) roll requested 2024-12-10T07:26:20,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580301 2024-12-10T07:26:20,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580301 2024-12-10T07:26:20,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580301) roll requested 2024-12-10T07:26:20,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580306 2024-12-10T07:26:20,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580306 2024-12-10T07:26:20,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580306) roll requested 2024-12-10T07:26:20,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580311 2024-12-10T07:26:20,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580311 2024-12-10T07:26:20,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580311) roll requested 2024-12-10T07:26:20,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580318 2024-12-10T07:26:20,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580318 2024-12-10T07:26:20,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580318) roll requested 2024-12-10T07:26:20,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580324 2024-12-10T07:26:20,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580324 2024-12-10T07:26:20,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580324) roll requested 2024-12-10T07:26:20,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580332 2024-12-10T07:26:20,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580332 2024-12-10T07:26:20,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580332) roll requested 2024-12-10T07:26:20,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580342 2024-12-10T07:26:20,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580342 2024-12-10T07:26:20,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580342) roll requested 2024-12-10T07:26:20,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580349 2024-12-10T07:26:20,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580349 2024-12-10T07:26:20,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580349) roll requested 2024-12-10T07:26:20,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580354 2024-12-10T07:26:20,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580354 2024-12-10T07:26:20,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580354) roll requested 2024-12-10T07:26:20,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580361 2024-12-10T07:26:20,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580361 2024-12-10T07:26:20,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580361) roll requested 2024-12-10T07:26:20,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580368 2024-12-10T07:26:20,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580368 2024-12-10T07:26:20,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580368) roll requested 2024-12-10T07:26:20,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580374 2024-12-10T07:26:20,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580374 2024-12-10T07:26:20,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580374) roll requested 2024-12-10T07:26:20,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580380 2024-12-10T07:26:20,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580380 2024-12-10T07:26:20,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580380) roll requested 2024-12-10T07:26:20,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580386 2024-12-10T07:26:20,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580386 2024-12-10T07:26:20,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580386) roll requested 2024-12-10T07:26:20,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580393 2024-12-10T07:26:20,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580393 2024-12-10T07:26:20,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580393) roll requested 2024-12-10T07:26:20,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580401 2024-12-10T07:26:20,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580401 2024-12-10T07:26:20,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580401) roll requested 2024-12-10T07:26:20,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580409 2024-12-10T07:26:20,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580409 2024-12-10T07:26:20,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580409) roll requested 2024-12-10T07:26:20,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580419 2024-12-10T07:26:20,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580419 2024-12-10T07:26:20,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580419) roll requested 2024-12-10T07:26:20,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580427 2024-12-10T07:26:20,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580427 2024-12-10T07:26:20,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580427) roll requested 2024-12-10T07:26:20,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580434 2024-12-10T07:26:20,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580434 2024-12-10T07:26:20,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580434) roll requested 2024-12-10T07:26:20,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580442 2024-12-10T07:26:20,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580442 2024-12-10T07:26:20,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580442) roll requested 2024-12-10T07:26:20,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580451 2024-12-10T07:26:20,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580451 2024-12-10T07:26:20,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580451) roll requested 2024-12-10T07:26:20,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580460 2024-12-10T07:26:20,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580460 2024-12-10T07:26:20,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580460) roll requested 2024-12-10T07:26:20,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580467 2024-12-10T07:26:20,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580467 2024-12-10T07:26:20,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580467) roll requested 2024-12-10T07:26:20,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580475 2024-12-10T07:26:20,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580475 2024-12-10T07:26:20,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580475) roll requested 2024-12-10T07:26:20,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580483 2024-12-10T07:26:20,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580483 2024-12-10T07:26:20,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580483) roll requested 2024-12-10T07:26:20,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580492 2024-12-10T07:26:20,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580492 2024-12-10T07:26:20,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580492) roll requested 2024-12-10T07:26:20,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580500 2024-12-10T07:26:20,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580500 2024-12-10T07:26:20,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580500) roll requested 2024-12-10T07:26:20,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580507 2024-12-10T07:26:20,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580507 2024-12-10T07:26:20,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580507) roll requested 2024-12-10T07:26:20,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580514 2024-12-10T07:26:20,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580514 2024-12-10T07:26:20,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580514) roll requested 2024-12-10T07:26:20,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580522 2024-12-10T07:26:20,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580522 2024-12-10T07:26:20,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580522) roll requested 2024-12-10T07:26:20,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580530 2024-12-10T07:26:20,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580530 2024-12-10T07:26:20,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580530) roll requested 2024-12-10T07:26:20,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580538 2024-12-10T07:26:20,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580538 2024-12-10T07:26:20,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580538) roll requested 2024-12-10T07:26:20,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580548 2024-12-10T07:26:20,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580548 2024-12-10T07:26:20,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580548) roll requested 2024-12-10T07:26:20,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580558 2024-12-10T07:26:20,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580558 2024-12-10T07:26:20,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580558) roll requested 2024-12-10T07:26:20,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580568 2024-12-10T07:26:20,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580568 2024-12-10T07:26:20,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580568) roll requested 2024-12-10T07:26:20,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580576 2024-12-10T07:26:20,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580576 2024-12-10T07:26:20,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580576) roll requested 2024-12-10T07:26:20,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580585 2024-12-10T07:26:20,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580585 2024-12-10T07:26:20,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580585) roll requested 2024-12-10T07:26:20,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580597 2024-12-10T07:26:20,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580597 2024-12-10T07:26:20,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580597) roll requested 2024-12-10T07:26:20,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580604 2024-12-10T07:26:20,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580604 2024-12-10T07:26:20,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580604) roll requested 2024-12-10T07:26:20,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580609 2024-12-10T07:26:20,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580609 2024-12-10T07:26:20,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580609) roll requested 2024-12-10T07:26:20,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580615 2024-12-10T07:26:20,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580615 2024-12-10T07:26:20,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580615) roll requested 2024-12-10T07:26:20,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580622 2024-12-10T07:26:20,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580622 2024-12-10T07:26:20,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580622) roll requested 2024-12-10T07:26:20,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580630 2024-12-10T07:26:20,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580630 2024-12-10T07:26:20,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580630) roll requested 2024-12-10T07:26:20,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580638 2024-12-10T07:26:20,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580638 2024-12-10T07:26:20,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580638) roll requested 2024-12-10T07:26:20,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580645 2024-12-10T07:26:20,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580645 2024-12-10T07:26:20,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580645) roll requested 2024-12-10T07:26:20,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580654 2024-12-10T07:26:20,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580654 2024-12-10T07:26:20,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580654) roll requested 2024-12-10T07:26:20,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580661 2024-12-10T07:26:20,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580661 2024-12-10T07:26:20,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580661) roll requested 2024-12-10T07:26:20,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580668 2024-12-10T07:26:20,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580668 2024-12-10T07:26:20,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580668) roll requested 2024-12-10T07:26:20,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580675 2024-12-10T07:26:20,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580675 2024-12-10T07:26:20,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580675) roll requested 2024-12-10T07:26:20,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580682 2024-12-10T07:26:20,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580682 2024-12-10T07:26:20,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580682) roll requested 2024-12-10T07:26:20,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580689 2024-12-10T07:26:20,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580689 2024-12-10T07:26:20,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580689) roll requested 2024-12-10T07:26:20,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580698 2024-12-10T07:26:20,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580698 2024-12-10T07:26:20,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580698) roll requested 2024-12-10T07:26:20,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580705 2024-12-10T07:26:20,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580705 2024-12-10T07:26:20,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580705) roll requested 2024-12-10T07:26:20,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580713 2024-12-10T07:26:20,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580713 2024-12-10T07:26:20,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580713) roll requested 2024-12-10T07:26:20,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580723 2024-12-10T07:26:20,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580723 2024-12-10T07:26:20,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580723) roll requested 2024-12-10T07:26:20,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580729 2024-12-10T07:26:20,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580729 2024-12-10T07:26:20,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580729) roll requested 2024-12-10T07:26:20,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580734 2024-12-10T07:26:20,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580734 2024-12-10T07:26:20,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580734) roll requested 2024-12-10T07:26:20,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580739 2024-12-10T07:26:20,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580739 2024-12-10T07:26:20,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580739) roll requested 2024-12-10T07:26:20,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580746 2024-12-10T07:26:20,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580746 2024-12-10T07:26:20,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580746) roll requested 2024-12-10T07:26:20,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580753 2024-12-10T07:26:20,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580753 2024-12-10T07:26:20,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580753) roll requested 2024-12-10T07:26:20,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580758 2024-12-10T07:26:20,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580758 2024-12-10T07:26:20,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580758) roll requested 2024-12-10T07:26:20,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580764 2024-12-10T07:26:20,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580764 2024-12-10T07:26:20,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580764) roll requested 2024-12-10T07:26:20,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580770 2024-12-10T07:26:20,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580770 2024-12-10T07:26:20,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580770) roll requested 2024-12-10T07:26:20,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580780 2024-12-10T07:26:20,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580780 2024-12-10T07:26:20,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580780) roll requested 2024-12-10T07:26:20,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580788 2024-12-10T07:26:20,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580788 2024-12-10T07:26:20,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580788) roll requested 2024-12-10T07:26:20,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580798 2024-12-10T07:26:20,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580798 2024-12-10T07:26:20,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580798) roll requested 2024-12-10T07:26:20,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580803 2024-12-10T07:26:20,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580803 2024-12-10T07:26:20,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580803) roll requested 2024-12-10T07:26:20,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580808 2024-12-10T07:26:20,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580808 2024-12-10T07:26:20,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580808) roll requested 2024-12-10T07:26:20,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580814 2024-12-10T07:26:20,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580814 2024-12-10T07:26:20,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580814) roll requested 2024-12-10T07:26:20,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580820 2024-12-10T07:26:20,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580820 2024-12-10T07:26:20,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580820) roll requested 2024-12-10T07:26:20,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580828 2024-12-10T07:26:20,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580828 2024-12-10T07:26:20,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580828) roll requested 2024-12-10T07:26:20,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580836 2024-12-10T07:26:20,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580836 2024-12-10T07:26:20,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580836) roll requested 2024-12-10T07:26:20,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580844 2024-12-10T07:26:20,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580844 2024-12-10T07:26:20,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580844) roll requested 2024-12-10T07:26:20,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580851 2024-12-10T07:26:20,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580851 2024-12-10T07:26:20,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580851) roll requested 2024-12-10T07:26:20,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580860 2024-12-10T07:26:20,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580860 2024-12-10T07:26:20,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580860) roll requested 2024-12-10T07:26:20,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580869 2024-12-10T07:26:20,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580869 2024-12-10T07:26:20,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580869) roll requested 2024-12-10T07:26:20,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580876 2024-12-10T07:26:20,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580876 2024-12-10T07:26:20,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580876) roll requested 2024-12-10T07:26:20,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580885 2024-12-10T07:26:20,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580885 2024-12-10T07:26:20,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580885) roll requested 2024-12-10T07:26:20,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580893 2024-12-10T07:26:20,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580893 2024-12-10T07:26:20,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580893) roll requested 2024-12-10T07:26:20,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580901 2024-12-10T07:26:20,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580901 2024-12-10T07:26:20,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580901) roll requested 2024-12-10T07:26:20,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580912 2024-12-10T07:26:20,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580912 2024-12-10T07:26:20,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580912) roll requested 2024-12-10T07:26:20,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580919 2024-12-10T07:26:20,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580919 2024-12-10T07:26:20,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580919) roll requested 2024-12-10T07:26:20,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580925 2024-12-10T07:26:20,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580925 2024-12-10T07:26:20,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580925) roll requested 2024-12-10T07:26:20,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580931 2024-12-10T07:26:20,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580931 2024-12-10T07:26:20,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580931) roll requested 2024-12-10T07:26:20,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580938 2024-12-10T07:26:20,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580938 2024-12-10T07:26:20,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580938) roll requested 2024-12-10T07:26:20,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580947 2024-12-10T07:26:20,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580947 2024-12-10T07:26:20,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580947) roll requested 2024-12-10T07:26:20,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580957 2024-12-10T07:26:20,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580957 2024-12-10T07:26:20,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580957) roll requested 2024-12-10T07:26:20,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580968 2024-12-10T07:26:20,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580968 2024-12-10T07:26:20,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580968) roll requested 2024-12-10T07:26:20,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580974 2024-12-10T07:26:20,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580974 2024-12-10T07:26:20,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580974) roll requested 2024-12-10T07:26:20,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580979 2024-12-10T07:26:20,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580979 2024-12-10T07:26:20,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580979) roll requested 2024-12-10T07:26:20,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580986 2024-12-10T07:26:20,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580986 2024-12-10T07:26:20,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:20,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:20,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580986) roll requested 2024-12-10T07:26:20,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815580994 2024-12-10T07:26:20,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:20,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580994 2024-12-10T07:26:21,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815580994) roll requested 2024-12-10T07:26:21,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581000 2024-12-10T07:26:21,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815580994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581000 2024-12-10T07:26:21,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581000) roll requested 2024-12-10T07:26:21,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581007 2024-12-10T07:26:21,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581007 2024-12-10T07:26:21,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581007) roll requested 2024-12-10T07:26:21,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581021 2024-12-10T07:26:21,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581021 2024-12-10T07:26:21,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581021) roll requested 2024-12-10T07:26:21,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581027 2024-12-10T07:26:21,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581027 2024-12-10T07:26:21,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581027) roll requested 2024-12-10T07:26:21,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581032 2024-12-10T07:26:21,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581032 2024-12-10T07:26:21,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581032) roll requested 2024-12-10T07:26:21,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581037 2024-12-10T07:26:21,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581037 2024-12-10T07:26:21,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581037) roll requested 2024-12-10T07:26:21,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581044 2024-12-10T07:26:21,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581044 2024-12-10T07:26:21,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581044) roll requested 2024-12-10T07:26:21,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581052 2024-12-10T07:26:21,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581052 2024-12-10T07:26:21,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581052) roll requested 2024-12-10T07:26:21,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581059 2024-12-10T07:26:21,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581059 2024-12-10T07:26:21,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581059) roll requested 2024-12-10T07:26:21,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581068 2024-12-10T07:26:21,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581068 2024-12-10T07:26:21,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581068) roll requested 2024-12-10T07:26:21,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581072 2024-12-10T07:26:21,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581072 2024-12-10T07:26:21,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581072) roll requested 2024-12-10T07:26:21,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581077 2024-12-10T07:26:21,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581077 2024-12-10T07:26:21,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581077) roll requested 2024-12-10T07:26:21,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581083 2024-12-10T07:26:21,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581083 2024-12-10T07:26:21,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581083) roll requested 2024-12-10T07:26:21,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581089 2024-12-10T07:26:21,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581089 2024-12-10T07:26:21,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581089) roll requested 2024-12-10T07:26:21,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581095 2024-12-10T07:26:21,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581095 2024-12-10T07:26:21,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581095) roll requested 2024-12-10T07:26:21,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581102 2024-12-10T07:26:21,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581102 2024-12-10T07:26:21,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581102) roll requested 2024-12-10T07:26:21,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581110 2024-12-10T07:26:21,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581110 2024-12-10T07:26:21,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581110) roll requested 2024-12-10T07:26:21,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581119 2024-12-10T07:26:21,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581119 2024-12-10T07:26:21,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581119) roll requested 2024-12-10T07:26:21,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581127 2024-12-10T07:26:21,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581127 2024-12-10T07:26:21,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581127) roll requested 2024-12-10T07:26:21,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581134 2024-12-10T07:26:21,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581134 2024-12-10T07:26:21,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581134) roll requested 2024-12-10T07:26:21,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581142 2024-12-10T07:26:21,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581142 2024-12-10T07:26:21,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581142) roll requested 2024-12-10T07:26:21,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581151 2024-12-10T07:26:21,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581151 2024-12-10T07:26:21,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581151) roll requested 2024-12-10T07:26:21,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581161 2024-12-10T07:26:21,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581161 2024-12-10T07:26:21,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581161) roll requested 2024-12-10T07:26:21,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581170 2024-12-10T07:26:21,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581170 2024-12-10T07:26:21,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581170) roll requested 2024-12-10T07:26:21,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581179 2024-12-10T07:26:21,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581179 2024-12-10T07:26:21,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581179) roll requested 2024-12-10T07:26:21,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581187 2024-12-10T07:26:21,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581187 2024-12-10T07:26:21,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581187) roll requested 2024-12-10T07:26:21,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581195 2024-12-10T07:26:21,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581195 2024-12-10T07:26:21,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581195) roll requested 2024-12-10T07:26:21,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581204 2024-12-10T07:26:21,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581204 2024-12-10T07:26:21,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581204) roll requested 2024-12-10T07:26:21,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581213 2024-12-10T07:26:21,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581213 2024-12-10T07:26:21,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581213) roll requested 2024-12-10T07:26:21,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581222 2024-12-10T07:26:21,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581222 2024-12-10T07:26:21,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581222) roll requested 2024-12-10T07:26:21,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581231 2024-12-10T07:26:21,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581231 2024-12-10T07:26:21,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581231) roll requested 2024-12-10T07:26:21,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581241 2024-12-10T07:26:21,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581241 2024-12-10T07:26:21,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581241) roll requested 2024-12-10T07:26:21,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581251 2024-12-10T07:26:21,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581251 2024-12-10T07:26:21,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581251) roll requested 2024-12-10T07:26:21,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581260 2024-12-10T07:26:21,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581260 2024-12-10T07:26:21,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581260) roll requested 2024-12-10T07:26:21,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581268 2024-12-10T07:26:21,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581268 2024-12-10T07:26:21,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581268) roll requested 2024-12-10T07:26:21,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581276 2024-12-10T07:26:21,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581276 2024-12-10T07:26:21,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581276) roll requested 2024-12-10T07:26:21,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581281 2024-12-10T07:26:21,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581281 2024-12-10T07:26:21,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581281) roll requested 2024-12-10T07:26:21,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581287 2024-12-10T07:26:21,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581287 2024-12-10T07:26:21,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581287) roll requested 2024-12-10T07:26:21,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581293 2024-12-10T07:26:21,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581293 2024-12-10T07:26:21,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581293) roll requested 2024-12-10T07:26:21,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581298 2024-12-10T07:26:21,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581298 2024-12-10T07:26:21,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581298) roll requested 2024-12-10T07:26:21,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581303 2024-12-10T07:26:21,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581303 2024-12-10T07:26:21,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581303) roll requested 2024-12-10T07:26:21,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581308 2024-12-10T07:26:21,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581308 2024-12-10T07:26:21,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581308) roll requested 2024-12-10T07:26:21,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581315 2024-12-10T07:26:21,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581315 2024-12-10T07:26:21,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581315) roll requested 2024-12-10T07:26:21,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581324 2024-12-10T07:26:21,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581324 2024-12-10T07:26:21,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581324) roll requested 2024-12-10T07:26:21,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581332 2024-12-10T07:26:21,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581332 2024-12-10T07:26:21,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581332) roll requested 2024-12-10T07:26:21,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581338 2024-12-10T07:26:21,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581338 2024-12-10T07:26:21,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581338) roll requested 2024-12-10T07:26:21,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581347 2024-12-10T07:26:21,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581347 2024-12-10T07:26:21,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581347) roll requested 2024-12-10T07:26:21,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581356 2024-12-10T07:26:21,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581356 2024-12-10T07:26:21,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581356) roll requested 2024-12-10T07:26:21,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581366 2024-12-10T07:26:21,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581366 2024-12-10T07:26:21,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581366) roll requested 2024-12-10T07:26:21,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581374 2024-12-10T07:26:21,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581374 2024-12-10T07:26:21,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581374) roll requested 2024-12-10T07:26:21,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581383 2024-12-10T07:26:21,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581383 2024-12-10T07:26:21,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581383) roll requested 2024-12-10T07:26:21,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581392 2024-12-10T07:26:21,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581392 2024-12-10T07:26:21,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581392) roll requested 2024-12-10T07:26:21,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581401 2024-12-10T07:26:21,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581401 2024-12-10T07:26:21,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581401) roll requested 2024-12-10T07:26:21,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581411 2024-12-10T07:26:21,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581411 2024-12-10T07:26:21,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581411) roll requested 2024-12-10T07:26:21,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581418 2024-12-10T07:26:21,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581418 2024-12-10T07:26:21,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581418) roll requested 2024-12-10T07:26:21,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581424 2024-12-10T07:26:21,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581424 2024-12-10T07:26:21,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581424) roll requested 2024-12-10T07:26:21,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581432 2024-12-10T07:26:21,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581432 2024-12-10T07:26:21,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581432) roll requested 2024-12-10T07:26:21,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581439 2024-12-10T07:26:21,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581439 2024-12-10T07:26:21,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581439) roll requested 2024-12-10T07:26:21,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581447 2024-12-10T07:26:21,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581447 2024-12-10T07:26:21,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581447) roll requested 2024-12-10T07:26:21,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581456 2024-12-10T07:26:21,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581456 2024-12-10T07:26:21,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581456) roll requested 2024-12-10T07:26:21,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581465 2024-12-10T07:26:21,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581465 2024-12-10T07:26:21,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581465) roll requested 2024-12-10T07:26:21,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581473 2024-12-10T07:26:21,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581473 2024-12-10T07:26:21,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581473) roll requested 2024-12-10T07:26:21,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581480 2024-12-10T07:26:21,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581480 2024-12-10T07:26:21,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581480) roll requested 2024-12-10T07:26:21,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581487 2024-12-10T07:26:21,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581487 2024-12-10T07:26:21,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581487) roll requested 2024-12-10T07:26:21,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581494 2024-12-10T07:26:21,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581494 2024-12-10T07:26:21,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581494) roll requested 2024-12-10T07:26:21,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581502 2024-12-10T07:26:21,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581502 2024-12-10T07:26:21,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581502) roll requested 2024-12-10T07:26:21,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581509 2024-12-10T07:26:21,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581509 2024-12-10T07:26:21,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581509) roll requested 2024-12-10T07:26:21,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581518 2024-12-10T07:26:21,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581518 2024-12-10T07:26:21,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581518) roll requested 2024-12-10T07:26:21,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581526 2024-12-10T07:26:21,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581526 2024-12-10T07:26:21,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581526) roll requested 2024-12-10T07:26:21,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581532 2024-12-10T07:26:21,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581532 2024-12-10T07:26:21,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581532) roll requested 2024-12-10T07:26:21,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581539 2024-12-10T07:26:21,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581539 2024-12-10T07:26:21,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581539) roll requested 2024-12-10T07:26:21,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581547 2024-12-10T07:26:21,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581547 2024-12-10T07:26:21,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581547) roll requested 2024-12-10T07:26:21,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581555 2024-12-10T07:26:21,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581555 2024-12-10T07:26:21,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581555) roll requested 2024-12-10T07:26:21,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581565 2024-12-10T07:26:21,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581565 2024-12-10T07:26:21,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581565) roll requested 2024-12-10T07:26:21,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581573 2024-12-10T07:26:21,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581573 2024-12-10T07:26:21,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581573) roll requested 2024-12-10T07:26:21,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581581 2024-12-10T07:26:21,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581581 2024-12-10T07:26:21,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581581) roll requested 2024-12-10T07:26:21,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581588 2024-12-10T07:26:21,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581588 2024-12-10T07:26:21,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581588) roll requested 2024-12-10T07:26:21,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581594 2024-12-10T07:26:21,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581594 2024-12-10T07:26:21,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581594) roll requested 2024-12-10T07:26:21,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581601 2024-12-10T07:26:21,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581601 2024-12-10T07:26:21,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581601) roll requested 2024-12-10T07:26:21,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581608 2024-12-10T07:26:21,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581608 2024-12-10T07:26:21,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581608) roll requested 2024-12-10T07:26:21,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581616 2024-12-10T07:26:21,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581616 2024-12-10T07:26:21,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581616) roll requested 2024-12-10T07:26:21,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581625 2024-12-10T07:26:21,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581625 2024-12-10T07:26:21,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581625) roll requested 2024-12-10T07:26:21,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581634 2024-12-10T07:26:21,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581634 2024-12-10T07:26:21,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581634) roll requested 2024-12-10T07:26:21,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581641 2024-12-10T07:26:21,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581641 2024-12-10T07:26:21,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581641) roll requested 2024-12-10T07:26:21,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581648 2024-12-10T07:26:21,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581648 2024-12-10T07:26:21,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581648) roll requested 2024-12-10T07:26:21,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581656 2024-12-10T07:26:21,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581656 2024-12-10T07:26:21,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581656) roll requested 2024-12-10T07:26:21,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581665 2024-12-10T07:26:21,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581665 2024-12-10T07:26:21,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581665) roll requested 2024-12-10T07:26:21,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581675 2024-12-10T07:26:21,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581675 2024-12-10T07:26:21,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581675) roll requested 2024-12-10T07:26:21,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581684 2024-12-10T07:26:21,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581684 2024-12-10T07:26:21,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581684) roll requested 2024-12-10T07:26:21,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581692 2024-12-10T07:26:21,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581692 2024-12-10T07:26:21,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581692) roll requested 2024-12-10T07:26:21,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581700 2024-12-10T07:26:21,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581700 2024-12-10T07:26:21,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581700) roll requested 2024-12-10T07:26:21,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581705 2024-12-10T07:26:21,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581705 2024-12-10T07:26:21,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581705) roll requested 2024-12-10T07:26:21,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581710 2024-12-10T07:26:21,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581710 2024-12-10T07:26:21,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581710) roll requested 2024-12-10T07:26:21,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581716 2024-12-10T07:26:21,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581716 2024-12-10T07:26:21,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581716) roll requested 2024-12-10T07:26:21,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581721 2024-12-10T07:26:21,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581721 2024-12-10T07:26:21,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581721) roll requested 2024-12-10T07:26:21,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581730 2024-12-10T07:26:21,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581730 2024-12-10T07:26:21,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581730) roll requested 2024-12-10T07:26:21,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581736 2024-12-10T07:26:21,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581736 2024-12-10T07:26:21,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581736) roll requested 2024-12-10T07:26:21,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581741 2024-12-10T07:26:21,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581741 2024-12-10T07:26:21,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581741) roll requested 2024-12-10T07:26:21,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581746 2024-12-10T07:26:21,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581746 2024-12-10T07:26:21,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581746) roll requested 2024-12-10T07:26:21,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581752 2024-12-10T07:26:21,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581752 2024-12-10T07:26:21,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581752) roll requested 2024-12-10T07:26:21,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581761 2024-12-10T07:26:21,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581761 2024-12-10T07:26:21,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581761) roll requested 2024-12-10T07:26:21,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581771 2024-12-10T07:26:21,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581771 2024-12-10T07:26:21,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581771) roll requested 2024-12-10T07:26:21,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581781 2024-12-10T07:26:21,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581781 2024-12-10T07:26:21,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581781) roll requested 2024-12-10T07:26:21,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581786 2024-12-10T07:26:21,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581786 2024-12-10T07:26:21,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581786) roll requested 2024-12-10T07:26:21,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581796 2024-12-10T07:26:21,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581796 2024-12-10T07:26:21,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581796) roll requested 2024-12-10T07:26:21,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581801 2024-12-10T07:26:21,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581801 2024-12-10T07:26:21,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581801) roll requested 2024-12-10T07:26:21,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581806 2024-12-10T07:26:21,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581806 2024-12-10T07:26:21,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581806) roll requested 2024-12-10T07:26:21,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581812 2024-12-10T07:26:21,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581812 2024-12-10T07:26:21,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581812) roll requested 2024-12-10T07:26:21,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581823 2024-12-10T07:26:21,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581823 2024-12-10T07:26:21,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581823) roll requested 2024-12-10T07:26:21,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581834 2024-12-10T07:26:21,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581834 2024-12-10T07:26:21,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581834) roll requested 2024-12-10T07:26:21,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581842 2024-12-10T07:26:21,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581842 2024-12-10T07:26:21,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581842) roll requested 2024-12-10T07:26:21,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581849 2024-12-10T07:26:21,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581849 2024-12-10T07:26:21,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581849) roll requested 2024-12-10T07:26:21,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581854 2024-12-10T07:26:21,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581854 2024-12-10T07:26:21,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581854) roll requested 2024-12-10T07:26:21,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581859 2024-12-10T07:26:21,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581859 2024-12-10T07:26:21,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581859) roll requested 2024-12-10T07:26:21,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581866 2024-12-10T07:26:21,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581866 2024-12-10T07:26:21,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581866) roll requested 2024-12-10T07:26:21,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581872 2024-12-10T07:26:21,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581872 2024-12-10T07:26:21,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581872) roll requested 2024-12-10T07:26:21,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581878 2024-12-10T07:26:21,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581878 2024-12-10T07:26:21,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581878) roll requested 2024-12-10T07:26:21,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581884 2024-12-10T07:26:21,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581884 2024-12-10T07:26:21,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581884) roll requested 2024-12-10T07:26:21,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581889 2024-12-10T07:26:21,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581889 2024-12-10T07:26:21,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581889) roll requested 2024-12-10T07:26:21,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581894 2024-12-10T07:26:21,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581894 2024-12-10T07:26:21,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581894) roll requested 2024-12-10T07:26:21,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581900 2024-12-10T07:26:21,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581900 2024-12-10T07:26:21,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581900) roll requested 2024-12-10T07:26:21,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581905 2024-12-10T07:26:21,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581905 2024-12-10T07:26:21,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581905) roll requested 2024-12-10T07:26:21,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581911 2024-12-10T07:26:21,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581911 2024-12-10T07:26:21,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581911) roll requested 2024-12-10T07:26:21,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581916 2024-12-10T07:26:21,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581916 2024-12-10T07:26:21,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581916) roll requested 2024-12-10T07:26:21,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581922 2024-12-10T07:26:21,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581922 2024-12-10T07:26:21,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581922) roll requested 2024-12-10T07:26:21,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581928 2024-12-10T07:26:21,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581928 2024-12-10T07:26:21,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581928) roll requested 2024-12-10T07:26:21,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581934 2024-12-10T07:26:21,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581934 2024-12-10T07:26:21,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581934) roll requested 2024-12-10T07:26:21,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581941 2024-12-10T07:26:21,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581941 2024-12-10T07:26:21,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581941) roll requested 2024-12-10T07:26:21,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581948 2024-12-10T07:26:21,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581948 2024-12-10T07:26:21,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581948) roll requested 2024-12-10T07:26:21,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581953 2024-12-10T07:26:21,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581953 2024-12-10T07:26:21,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581953) roll requested 2024-12-10T07:26:21,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581959 2024-12-10T07:26:21,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581959 2024-12-10T07:26:21,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581959) roll requested 2024-12-10T07:26:21,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581966 2024-12-10T07:26:21,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581966 2024-12-10T07:26:21,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581966) roll requested 2024-12-10T07:26:21,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581973 2024-12-10T07:26:21,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581973 2024-12-10T07:26:21,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581973) roll requested 2024-12-10T07:26:21,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581980 2024-12-10T07:26:21,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581980 2024-12-10T07:26:21,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:21,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581980) roll requested 2024-12-10T07:26:21,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815581990 2024-12-10T07:26:21,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:21,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581990 2024-12-10T07:26:21,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:21,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815581990) roll requested 2024-12-10T07:26:22,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582000 2024-12-10T07:26:22,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815581990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582000 2024-12-10T07:26:22,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582000) roll requested 2024-12-10T07:26:22,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582008 2024-12-10T07:26:22,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582008 2024-12-10T07:26:22,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582008) roll requested 2024-12-10T07:26:22,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582014 2024-12-10T07:26:22,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582014 2024-12-10T07:26:22,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582014) roll requested 2024-12-10T07:26:22,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582020 2024-12-10T07:26:22,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582020 2024-12-10T07:26:22,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582020) roll requested 2024-12-10T07:26:22,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582026 2024-12-10T07:26:22,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582026 2024-12-10T07:26:22,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582026) roll requested 2024-12-10T07:26:22,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582033 2024-12-10T07:26:22,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582033 2024-12-10T07:26:22,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582033) roll requested 2024-12-10T07:26:22,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582039 2024-12-10T07:26:22,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582039 2024-12-10T07:26:22,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582039) roll requested 2024-12-10T07:26:22,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582048 2024-12-10T07:26:22,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582048 2024-12-10T07:26:22,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582048) roll requested 2024-12-10T07:26:22,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582057 2024-12-10T07:26:22,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582057 2024-12-10T07:26:22,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582057) roll requested 2024-12-10T07:26:22,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582068 2024-12-10T07:26:22,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582068 2024-12-10T07:26:22,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582068) roll requested 2024-12-10T07:26:22,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582077 2024-12-10T07:26:22,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582077 2024-12-10T07:26:22,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582077) roll requested 2024-12-10T07:26:22,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582086 2024-12-10T07:26:22,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582086 2024-12-10T07:26:22,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582086) roll requested 2024-12-10T07:26:22,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582095 2024-12-10T07:26:22,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582095 2024-12-10T07:26:22,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582095) roll requested 2024-12-10T07:26:22,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582105 2024-12-10T07:26:22,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582105 2024-12-10T07:26:22,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582105) roll requested 2024-12-10T07:26:22,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582112 2024-12-10T07:26:22,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582112 2024-12-10T07:26:22,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582112) roll requested 2024-12-10T07:26:22,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582119 2024-12-10T07:26:22,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582119 2024-12-10T07:26:22,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582119) roll requested 2024-12-10T07:26:22,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582127 2024-12-10T07:26:22,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582127 2024-12-10T07:26:22,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582127) roll requested 2024-12-10T07:26:22,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582136 2024-12-10T07:26:22,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582136 2024-12-10T07:26:22,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582136) roll requested 2024-12-10T07:26:22,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582144 2024-12-10T07:26:22,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582144 2024-12-10T07:26:22,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582144) roll requested 2024-12-10T07:26:22,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582153 2024-12-10T07:26:22,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582153 2024-12-10T07:26:22,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582153) roll requested 2024-12-10T07:26:22,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582161 2024-12-10T07:26:22,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582161 2024-12-10T07:26:22,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582161) roll requested 2024-12-10T07:26:22,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582169 2024-12-10T07:26:22,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582169 2024-12-10T07:26:22,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582169) roll requested 2024-12-10T07:26:22,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582177 2024-12-10T07:26:22,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582177 2024-12-10T07:26:22,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582177) roll requested 2024-12-10T07:26:22,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582184 2024-12-10T07:26:22,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582184 2024-12-10T07:26:22,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582184) roll requested 2024-12-10T07:26:22,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582190 2024-12-10T07:26:22,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582190 2024-12-10T07:26:22,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582190) roll requested 2024-12-10T07:26:22,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582197 2024-12-10T07:26:22,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582197 2024-12-10T07:26:22,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582197) roll requested 2024-12-10T07:26:22,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582204 2024-12-10T07:26:22,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582204 2024-12-10T07:26:22,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582204) roll requested 2024-12-10T07:26:22,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582213 2024-12-10T07:26:22,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582213 2024-12-10T07:26:22,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582213) roll requested 2024-12-10T07:26:22,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582221 2024-12-10T07:26:22,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582221 2024-12-10T07:26:22,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582221) roll requested 2024-12-10T07:26:22,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582228 2024-12-10T07:26:22,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582228 2024-12-10T07:26:22,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582228) roll requested 2024-12-10T07:26:22,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582235 2024-12-10T07:26:22,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582235 2024-12-10T07:26:22,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582235) roll requested 2024-12-10T07:26:22,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582242 2024-12-10T07:26:22,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582242 2024-12-10T07:26:22,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582242) roll requested 2024-12-10T07:26:22,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582250 2024-12-10T07:26:22,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582250 2024-12-10T07:26:22,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582250) roll requested 2024-12-10T07:26:22,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582259 2024-12-10T07:26:22,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582259 2024-12-10T07:26:22,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582259) roll requested 2024-12-10T07:26:22,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582272 2024-12-10T07:26:22,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582272 2024-12-10T07:26:22,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582272) roll requested 2024-12-10T07:26:22,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582279 2024-12-10T07:26:22,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582279 2024-12-10T07:26:22,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582279) roll requested 2024-12-10T07:26:22,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582284 2024-12-10T07:26:22,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582284 2024-12-10T07:26:22,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582284) roll requested 2024-12-10T07:26:22,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582289 2024-12-10T07:26:22,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582289 2024-12-10T07:26:22,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582289) roll requested 2024-12-10T07:26:22,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582294 2024-12-10T07:26:22,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582294 2024-12-10T07:26:22,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582294) roll requested 2024-12-10T07:26:22,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582300 2024-12-10T07:26:22,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582300 2024-12-10T07:26:22,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582300) roll requested 2024-12-10T07:26:22,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582307 2024-12-10T07:26:22,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582307 2024-12-10T07:26:22,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582307) roll requested 2024-12-10T07:26:22,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582314 2024-12-10T07:26:22,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582314 2024-12-10T07:26:22,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582314) roll requested 2024-12-10T07:26:22,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582323 2024-12-10T07:26:22,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582323 2024-12-10T07:26:22,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582323) roll requested 2024-12-10T07:26:22,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582329 2024-12-10T07:26:22,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582329 2024-12-10T07:26:22,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582329) roll requested 2024-12-10T07:26:22,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582337 2024-12-10T07:26:22,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582337 2024-12-10T07:26:22,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582337) roll requested 2024-12-10T07:26:22,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582345 2024-12-10T07:26:22,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582345 2024-12-10T07:26:22,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582345) roll requested 2024-12-10T07:26:22,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582353 2024-12-10T07:26:22,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582353 2024-12-10T07:26:22,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582353) roll requested 2024-12-10T07:26:22,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582361 2024-12-10T07:26:22,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582361 2024-12-10T07:26:22,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582361) roll requested 2024-12-10T07:26:22,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582369 2024-12-10T07:26:22,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582369 2024-12-10T07:26:22,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582369) roll requested 2024-12-10T07:26:22,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582378 2024-12-10T07:26:22,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582378 2024-12-10T07:26:22,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582378) roll requested 2024-12-10T07:26:22,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582385 2024-12-10T07:26:22,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582385 2024-12-10T07:26:22,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582385) roll requested 2024-12-10T07:26:22,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582392 2024-12-10T07:26:22,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582392 2024-12-10T07:26:22,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582392) roll requested 2024-12-10T07:26:22,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582401 2024-12-10T07:26:22,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582401 2024-12-10T07:26:22,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582401) roll requested 2024-12-10T07:26:22,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582409 2024-12-10T07:26:22,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582409 2024-12-10T07:26:22,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582409) roll requested 2024-12-10T07:26:22,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582413 2024-12-10T07:26:22,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582413 2024-12-10T07:26:22,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582413) roll requested 2024-12-10T07:26:22,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582419 2024-12-10T07:26:22,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582419 2024-12-10T07:26:22,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582419) roll requested 2024-12-10T07:26:22,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582426 2024-12-10T07:26:22,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582426 2024-12-10T07:26:22,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582426) roll requested 2024-12-10T07:26:22,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582434 2024-12-10T07:26:22,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582434 2024-12-10T07:26:22,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582434) roll requested 2024-12-10T07:26:22,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582442 2024-12-10T07:26:22,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582442 2024-12-10T07:26:22,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582442) roll requested 2024-12-10T07:26:22,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582447 2024-12-10T07:26:22,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582447 2024-12-10T07:26:22,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582447) roll requested 2024-12-10T07:26:22,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582454 2024-12-10T07:26:22,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582454 2024-12-10T07:26:22,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582454) roll requested 2024-12-10T07:26:22,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582461 2024-12-10T07:26:22,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582461 2024-12-10T07:26:22,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582461) roll requested 2024-12-10T07:26:22,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582469 2024-12-10T07:26:22,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582469 2024-12-10T07:26:22,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582469) roll requested 2024-12-10T07:26:22,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582477 2024-12-10T07:26:22,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582477 2024-12-10T07:26:22,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582477) roll requested 2024-12-10T07:26:22,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582487 2024-12-10T07:26:22,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582487 2024-12-10T07:26:22,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582487) roll requested 2024-12-10T07:26:22,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582497 2024-12-10T07:26:22,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582497 2024-12-10T07:26:22,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582497) roll requested 2024-12-10T07:26:22,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582508 2024-12-10T07:26:22,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582508 2024-12-10T07:26:22,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582508) roll requested 2024-12-10T07:26:22,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582515 2024-12-10T07:26:22,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582515 2024-12-10T07:26:22,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582515) roll requested 2024-12-10T07:26:22,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582525 2024-12-10T07:26:22,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582525 2024-12-10T07:26:22,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582525) roll requested 2024-12-10T07:26:22,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582530 2024-12-10T07:26:22,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582530 2024-12-10T07:26:22,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582530) roll requested 2024-12-10T07:26:22,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582535 2024-12-10T07:26:22,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582535 2024-12-10T07:26:22,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582535) roll requested 2024-12-10T07:26:22,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582541 2024-12-10T07:26:22,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582541 2024-12-10T07:26:22,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582541) roll requested 2024-12-10T07:26:22,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582547 2024-12-10T07:26:22,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582547 2024-12-10T07:26:22,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582547) roll requested 2024-12-10T07:26:22,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582555 2024-12-10T07:26:22,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582555 2024-12-10T07:26:22,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582555) roll requested 2024-12-10T07:26:22,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582563 2024-12-10T07:26:22,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582563 2024-12-10T07:26:22,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582563) roll requested 2024-12-10T07:26:22,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582571 2024-12-10T07:26:22,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582571 2024-12-10T07:26:22,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582571) roll requested 2024-12-10T07:26:22,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582579 2024-12-10T07:26:22,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582579 2024-12-10T07:26:22,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582579) roll requested 2024-12-10T07:26:22,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582587 2024-12-10T07:26:22,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582587 2024-12-10T07:26:22,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582587) roll requested 2024-12-10T07:26:22,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582594 2024-12-10T07:26:22,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582594 2024-12-10T07:26:22,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582594) roll requested 2024-12-10T07:26:22,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582601 2024-12-10T07:26:22,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582601 2024-12-10T07:26:22,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582601) roll requested 2024-12-10T07:26:22,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582610 2024-12-10T07:26:22,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582610 2024-12-10T07:26:22,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582610) roll requested 2024-12-10T07:26:22,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582618 2024-12-10T07:26:22,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582618 2024-12-10T07:26:22,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582618) roll requested 2024-12-10T07:26:22,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582627 2024-12-10T07:26:22,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582627 2024-12-10T07:26:22,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582627) roll requested 2024-12-10T07:26:22,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582635 2024-12-10T07:26:22,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582635 2024-12-10T07:26:22,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582635) roll requested 2024-12-10T07:26:22,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582644 2024-12-10T07:26:22,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582644 2024-12-10T07:26:22,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582644) roll requested 2024-12-10T07:26:22,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582655 2024-12-10T07:26:22,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582655 2024-12-10T07:26:22,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582655) roll requested 2024-12-10T07:26:22,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582664 2024-12-10T07:26:22,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582664 2024-12-10T07:26:22,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582664) roll requested 2024-12-10T07:26:22,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582674 2024-12-10T07:26:22,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582674 2024-12-10T07:26:22,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582674) roll requested 2024-12-10T07:26:22,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582684 2024-12-10T07:26:22,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582684 2024-12-10T07:26:22,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582684) roll requested 2024-12-10T07:26:22,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582693 2024-12-10T07:26:22,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582693 2024-12-10T07:26:22,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582693) roll requested 2024-12-10T07:26:22,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582698 2024-12-10T07:26:22,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582698 2024-12-10T07:26:22,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582698) roll requested 2024-12-10T07:26:22,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582706 2024-12-10T07:26:22,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582706 2024-12-10T07:26:22,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582706) roll requested 2024-12-10T07:26:22,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582713 2024-12-10T07:26:22,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582713 2024-12-10T07:26:22,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582713) roll requested 2024-12-10T07:26:22,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582719 2024-12-10T07:26:22,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582719 2024-12-10T07:26:22,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582719) roll requested 2024-12-10T07:26:22,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582727 2024-12-10T07:26:22,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582727 2024-12-10T07:26:22,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582727) roll requested 2024-12-10T07:26:22,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582735 2024-12-10T07:26:22,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582735 2024-12-10T07:26:22,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582735) roll requested 2024-12-10T07:26:22,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582745 2024-12-10T07:26:22,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582745 2024-12-10T07:26:22,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582745) roll requested 2024-12-10T07:26:22,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582753 2024-12-10T07:26:22,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582753 2024-12-10T07:26:22,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582753) roll requested 2024-12-10T07:26:22,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582759 2024-12-10T07:26:22,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582759 2024-12-10T07:26:22,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582759) roll requested 2024-12-10T07:26:22,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582765 2024-12-10T07:26:22,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582765 2024-12-10T07:26:22,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582765) roll requested 2024-12-10T07:26:22,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582770 2024-12-10T07:26:22,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582770 2024-12-10T07:26:22,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582770) roll requested 2024-12-10T07:26:22,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582775 2024-12-10T07:26:22,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582775 2024-12-10T07:26:22,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582775) roll requested 2024-12-10T07:26:22,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582783 2024-12-10T07:26:22,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582783 2024-12-10T07:26:22,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582783) roll requested 2024-12-10T07:26:22,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582792 2024-12-10T07:26:22,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582792 2024-12-10T07:26:22,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582792) roll requested 2024-12-10T07:26:22,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582798 2024-12-10T07:26:22,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582798 2024-12-10T07:26:22,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582798) roll requested 2024-12-10T07:26:22,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582803 2024-12-10T07:26:22,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582803 2024-12-10T07:26:22,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582803) roll requested 2024-12-10T07:26:22,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582808 2024-12-10T07:26:22,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582808 2024-12-10T07:26:22,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582808) roll requested 2024-12-10T07:26:22,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582814 2024-12-10T07:26:22,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582814 2024-12-10T07:26:22,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582814) roll requested 2024-12-10T07:26:22,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582821 2024-12-10T07:26:22,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582821 2024-12-10T07:26:22,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582821) roll requested 2024-12-10T07:26:22,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582826 2024-12-10T07:26:22,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582826 2024-12-10T07:26:22,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582826) roll requested 2024-12-10T07:26:22,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582833 2024-12-10T07:26:22,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582833 2024-12-10T07:26:22,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582833) roll requested 2024-12-10T07:26:22,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582844 2024-12-10T07:26:22,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582844 2024-12-10T07:26:22,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582844) roll requested 2024-12-10T07:26:22,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582850 2024-12-10T07:26:22,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582850 2024-12-10T07:26:22,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582850) roll requested 2024-12-10T07:26:22,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582855 2024-12-10T07:26:22,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582855 2024-12-10T07:26:22,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582855) roll requested 2024-12-10T07:26:22,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582860 2024-12-10T07:26:22,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582860 2024-12-10T07:26:22,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582860) roll requested 2024-12-10T07:26:22,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582865 2024-12-10T07:26:22,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582865 2024-12-10T07:26:22,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582865) roll requested 2024-12-10T07:26:22,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582870 2024-12-10T07:26:22,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582870 2024-12-10T07:26:22,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582870) roll requested 2024-12-10T07:26:22,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582876 2024-12-10T07:26:22,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582876 2024-12-10T07:26:22,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582876) roll requested 2024-12-10T07:26:22,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582882 2024-12-10T07:26:22,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582882 2024-12-10T07:26:22,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582882) roll requested 2024-12-10T07:26:22,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582889 2024-12-10T07:26:22,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582889 2024-12-10T07:26:22,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582889) roll requested 2024-12-10T07:26:22,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582897 2024-12-10T07:26:22,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582897 2024-12-10T07:26:22,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582897) roll requested 2024-12-10T07:26:22,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582905 2024-12-10T07:26:22,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582905 2024-12-10T07:26:22,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582905) roll requested 2024-12-10T07:26:22,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582914 2024-12-10T07:26:22,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582914 2024-12-10T07:26:22,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582914) roll requested 2024-12-10T07:26:22,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582921 2024-12-10T07:26:22,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582921 2024-12-10T07:26:22,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582921) roll requested 2024-12-10T07:26:22,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582928 2024-12-10T07:26:22,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582928 2024-12-10T07:26:22,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582928) roll requested 2024-12-10T07:26:22,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582933 2024-12-10T07:26:22,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582933 2024-12-10T07:26:22,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582933) roll requested 2024-12-10T07:26:22,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582938 2024-12-10T07:26:22,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582938 2024-12-10T07:26:22,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582938) roll requested 2024-12-10T07:26:22,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582945 2024-12-10T07:26:22,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582945 2024-12-10T07:26:22,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582945) roll requested 2024-12-10T07:26:22,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582953 2024-12-10T07:26:22,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582953 2024-12-10T07:26:22,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582953) roll requested 2024-12-10T07:26:22,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582961 2024-12-10T07:26:22,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582961 2024-12-10T07:26:22,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582961) roll requested 2024-12-10T07:26:22,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582969 2024-12-10T07:26:22,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582969 2024-12-10T07:26:22,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582969) roll requested 2024-12-10T07:26:22,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582975 2024-12-10T07:26:22,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582975 2024-12-10T07:26:22,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582975) roll requested 2024-12-10T07:26:22,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582981 2024-12-10T07:26:22,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582981 2024-12-10T07:26:22,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582981) roll requested 2024-12-10T07:26:22,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582989 2024-12-10T07:26:22,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:22,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582989 2024-12-10T07:26:22,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:22,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:22,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582989) roll requested 2024-12-10T07:26:22,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815582998 2024-12-10T07:26:23,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582998 2024-12-10T07:26:23,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815582998) roll requested 2024-12-10T07:26:23,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583006 2024-12-10T07:26:23,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815582998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583006 2024-12-10T07:26:23,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583006) roll requested 2024-12-10T07:26:23,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583015 2024-12-10T07:26:23,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583015 2024-12-10T07:26:23,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583015) roll requested 2024-12-10T07:26:23,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583022 2024-12-10T07:26:23,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583022 2024-12-10T07:26:23,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583022) roll requested 2024-12-10T07:26:23,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583028 2024-12-10T07:26:23,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583028 2024-12-10T07:26:23,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583028) roll requested 2024-12-10T07:26:23,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583036 2024-12-10T07:26:23,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583036 2024-12-10T07:26:23,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583036) roll requested 2024-12-10T07:26:23,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583043 2024-12-10T07:26:23,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583043 2024-12-10T07:26:23,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583043) roll requested 2024-12-10T07:26:23,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583049 2024-12-10T07:26:23,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583049 2024-12-10T07:26:23,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583049) roll requested 2024-12-10T07:26:23,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583057 2024-12-10T07:26:23,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583057 2024-12-10T07:26:23,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583057) roll requested 2024-12-10T07:26:23,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583064 2024-12-10T07:26:23,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583064 2024-12-10T07:26:23,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583064) roll requested 2024-12-10T07:26:23,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583072 2024-12-10T07:26:23,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583072 2024-12-10T07:26:23,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583072) roll requested 2024-12-10T07:26:23,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583079 2024-12-10T07:26:23,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583079 2024-12-10T07:26:23,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583079) roll requested 2024-12-10T07:26:23,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583087 2024-12-10T07:26:23,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583087 2024-12-10T07:26:23,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583087) roll requested 2024-12-10T07:26:23,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583096 2024-12-10T07:26:23,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583096 2024-12-10T07:26:23,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583096) roll requested 2024-12-10T07:26:23,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583104 2024-12-10T07:26:23,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583104 2024-12-10T07:26:23,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583104) roll requested 2024-12-10T07:26:23,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583109 2024-12-10T07:26:23,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583109 2024-12-10T07:26:23,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583109) roll requested 2024-12-10T07:26:23,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583114 2024-12-10T07:26:23,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583114 2024-12-10T07:26:23,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583114) roll requested 2024-12-10T07:26:23,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583120 2024-12-10T07:26:23,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583120 2024-12-10T07:26:23,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583120) roll requested 2024-12-10T07:26:23,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583127 2024-12-10T07:26:23,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583127 2024-12-10T07:26:23,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583127) roll requested 2024-12-10T07:26:23,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583133 2024-12-10T07:26:23,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583133 2024-12-10T07:26:23,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583133) roll requested 2024-12-10T07:26:23,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583140 2024-12-10T07:26:23,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583140 2024-12-10T07:26:23,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583140) roll requested 2024-12-10T07:26:23,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583148 2024-12-10T07:26:23,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583148 2024-12-10T07:26:23,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583148) roll requested 2024-12-10T07:26:23,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583157 2024-12-10T07:26:23,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583157 2024-12-10T07:26:23,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583157) roll requested 2024-12-10T07:26:23,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583165 2024-12-10T07:26:23,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583165 2024-12-10T07:26:23,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583165) roll requested 2024-12-10T07:26:23,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583176 2024-12-10T07:26:23,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583176 2024-12-10T07:26:23,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583176) roll requested 2024-12-10T07:26:23,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583184 2024-12-10T07:26:23,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583184 2024-12-10T07:26:23,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583184) roll requested 2024-12-10T07:26:23,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583190 2024-12-10T07:26:23,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583190 2024-12-10T07:26:23,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583190) roll requested 2024-12-10T07:26:23,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583196 2024-12-10T07:26:23,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583196 2024-12-10T07:26:23,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583196) roll requested 2024-12-10T07:26:23,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583204 2024-12-10T07:26:23,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583204 2024-12-10T07:26:23,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583204) roll requested 2024-12-10T07:26:23,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583210 2024-12-10T07:26:23,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583210 2024-12-10T07:26:23,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583210) roll requested 2024-12-10T07:26:23,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583216 2024-12-10T07:26:23,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583216 2024-12-10T07:26:23,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583216) roll requested 2024-12-10T07:26:23,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583223 2024-12-10T07:26:23,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583223 2024-12-10T07:26:23,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583223) roll requested 2024-12-10T07:26:23,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583230 2024-12-10T07:26:23,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583230 2024-12-10T07:26:23,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583230) roll requested 2024-12-10T07:26:23,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583238 2024-12-10T07:26:23,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583238 2024-12-10T07:26:23,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583238) roll requested 2024-12-10T07:26:23,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583246 2024-12-10T07:26:23,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583246 2024-12-10T07:26:23,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583246) roll requested 2024-12-10T07:26:23,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583252 2024-12-10T07:26:23,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583252 2024-12-10T07:26:23,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583252) roll requested 2024-12-10T07:26:23,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583259 2024-12-10T07:26:23,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583259 2024-12-10T07:26:23,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583259) roll requested 2024-12-10T07:26:23,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583268 2024-12-10T07:26:23,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583268 2024-12-10T07:26:23,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583268) roll requested 2024-12-10T07:26:23,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583275 2024-12-10T07:26:23,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583275 2024-12-10T07:26:23,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583275) roll requested 2024-12-10T07:26:23,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583283 2024-12-10T07:26:23,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583283 2024-12-10T07:26:23,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583283) roll requested 2024-12-10T07:26:23,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583291 2024-12-10T07:26:23,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583291 2024-12-10T07:26:23,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583291) roll requested 2024-12-10T07:26:23,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583301 2024-12-10T07:26:23,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583301 2024-12-10T07:26:23,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583301) roll requested 2024-12-10T07:26:23,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583309 2024-12-10T07:26:23,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583309 2024-12-10T07:26:23,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583309) roll requested 2024-12-10T07:26:23,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583316 2024-12-10T07:26:23,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583316 2024-12-10T07:26:23,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583316) roll requested 2024-12-10T07:26:23,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583325 2024-12-10T07:26:23,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583325 2024-12-10T07:26:23,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583325) roll requested 2024-12-10T07:26:23,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583333 2024-12-10T07:26:23,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583333 2024-12-10T07:26:23,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583333) roll requested 2024-12-10T07:26:23,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583339 2024-12-10T07:26:23,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583339 2024-12-10T07:26:23,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583339) roll requested 2024-12-10T07:26:23,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583347 2024-12-10T07:26:23,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583347 2024-12-10T07:26:23,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583347) roll requested 2024-12-10T07:26:23,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583356 2024-12-10T07:26:23,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583356 2024-12-10T07:26:23,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583356) roll requested 2024-12-10T07:26:23,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583365 2024-12-10T07:26:23,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583365 2024-12-10T07:26:23,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583365) roll requested 2024-12-10T07:26:23,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583371 2024-12-10T07:26:23,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583371 2024-12-10T07:26:23,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583371) roll requested 2024-12-10T07:26:23,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583378 2024-12-10T07:26:23,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583378 2024-12-10T07:26:23,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583378) roll requested 2024-12-10T07:26:23,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583386 2024-12-10T07:26:23,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583386 2024-12-10T07:26:23,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583386) roll requested 2024-12-10T07:26:23,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583395 2024-12-10T07:26:23,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583395 2024-12-10T07:26:23,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583395) roll requested 2024-12-10T07:26:23,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583403 2024-12-10T07:26:23,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583403 2024-12-10T07:26:23,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583403) roll requested 2024-12-10T07:26:23,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583412 2024-12-10T07:26:23,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583412 2024-12-10T07:26:23,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583412) roll requested 2024-12-10T07:26:23,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583421 2024-12-10T07:26:23,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583421 2024-12-10T07:26:23,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583421) roll requested 2024-12-10T07:26:23,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583428 2024-12-10T07:26:23,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583428 2024-12-10T07:26:23,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583428) roll requested 2024-12-10T07:26:23,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583437 2024-12-10T07:26:23,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583437 2024-12-10T07:26:23,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583437) roll requested 2024-12-10T07:26:23,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583446 2024-12-10T07:26:23,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583446 2024-12-10T07:26:23,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583446) roll requested 2024-12-10T07:26:23,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583456 2024-12-10T07:26:23,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583456 2024-12-10T07:26:23,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583456) roll requested 2024-12-10T07:26:23,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583462 2024-12-10T07:26:23,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583462 2024-12-10T07:26:23,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583462) roll requested 2024-12-10T07:26:23,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583468 2024-12-10T07:26:23,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583468 2024-12-10T07:26:23,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583468) roll requested 2024-12-10T07:26:23,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583477 2024-12-10T07:26:23,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583477 2024-12-10T07:26:23,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583477) roll requested 2024-12-10T07:26:23,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583486 2024-12-10T07:26:23,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583486 2024-12-10T07:26:23,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583486) roll requested 2024-12-10T07:26:23,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583495 2024-12-10T07:26:23,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583495 2024-12-10T07:26:23,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583495) roll requested 2024-12-10T07:26:23,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583505 2024-12-10T07:26:23,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583505 2024-12-10T07:26:23,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583505) roll requested 2024-12-10T07:26:23,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583513 2024-12-10T07:26:23,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583513 2024-12-10T07:26:23,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583513) roll requested 2024-12-10T07:26:23,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583519 2024-12-10T07:26:23,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583519 2024-12-10T07:26:23,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583519) roll requested 2024-12-10T07:26:23,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583525 2024-12-10T07:26:23,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583525 2024-12-10T07:26:23,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583525) roll requested 2024-12-10T07:26:23,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583532 2024-12-10T07:26:23,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583532 2024-12-10T07:26:23,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583532) roll requested 2024-12-10T07:26:23,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583539 2024-12-10T07:26:23,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583539 2024-12-10T07:26:23,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583539) roll requested 2024-12-10T07:26:23,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583546 2024-12-10T07:26:23,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583546 2024-12-10T07:26:23,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583546) roll requested 2024-12-10T07:26:23,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583555 2024-12-10T07:26:23,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583555 2024-12-10T07:26:23,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583555) roll requested 2024-12-10T07:26:23,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583564 2024-12-10T07:26:23,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583564 2024-12-10T07:26:23,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583564) roll requested 2024-12-10T07:26:23,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583571 2024-12-10T07:26:23,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583571 2024-12-10T07:26:23,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583571) roll requested 2024-12-10T07:26:23,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583580 2024-12-10T07:26:23,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583580 2024-12-10T07:26:23,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583580) roll requested 2024-12-10T07:26:23,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583590 2024-12-10T07:26:23,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583590 2024-12-10T07:26:23,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583590) roll requested 2024-12-10T07:26:23,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583599 2024-12-10T07:26:23,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583599 2024-12-10T07:26:23,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583599) roll requested 2024-12-10T07:26:23,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583607 2024-12-10T07:26:23,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583607 2024-12-10T07:26:23,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583607) roll requested 2024-12-10T07:26:23,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583616 2024-12-10T07:26:23,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583616 2024-12-10T07:26:23,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583616) roll requested 2024-12-10T07:26:23,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583626 2024-12-10T07:26:23,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583626 2024-12-10T07:26:23,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583626) roll requested 2024-12-10T07:26:23,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583635 2024-12-10T07:26:23,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583635 2024-12-10T07:26:23,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583635) roll requested 2024-12-10T07:26:23,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583642 2024-12-10T07:26:23,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583642 2024-12-10T07:26:23,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583642) roll requested 2024-12-10T07:26:23,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583650 2024-12-10T07:26:23,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583650 2024-12-10T07:26:23,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583650) roll requested 2024-12-10T07:26:23,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583658 2024-12-10T07:26:23,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583658 2024-12-10T07:26:23,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583658) roll requested 2024-12-10T07:26:23,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583667 2024-12-10T07:26:23,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583667 2024-12-10T07:26:23,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583667) roll requested 2024-12-10T07:26:23,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583674 2024-12-10T07:26:23,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583674 2024-12-10T07:26:23,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583674) roll requested 2024-12-10T07:26:23,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583679 2024-12-10T07:26:23,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583679 2024-12-10T07:26:23,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583679) roll requested 2024-12-10T07:26:23,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583685 2024-12-10T07:26:23,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583685 2024-12-10T07:26:23,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583685) roll requested 2024-12-10T07:26:23,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583697 2024-12-10T07:26:23,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583697 2024-12-10T07:26:23,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583697) roll requested 2024-12-10T07:26:23,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583702 2024-12-10T07:26:23,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583702 2024-12-10T07:26:23,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583702) roll requested 2024-12-10T07:26:23,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583709 2024-12-10T07:26:23,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583709 2024-12-10T07:26:23,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583709) roll requested 2024-12-10T07:26:23,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583714 2024-12-10T07:26:23,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583714 2024-12-10T07:26:23,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583714) roll requested 2024-12-10T07:26:23,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583721 2024-12-10T07:26:23,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583721 2024-12-10T07:26:23,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583721) roll requested 2024-12-10T07:26:23,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583726 2024-12-10T07:26:23,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583726 2024-12-10T07:26:23,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583726) roll requested 2024-12-10T07:26:23,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583732 2024-12-10T07:26:23,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583732 2024-12-10T07:26:23,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583732) roll requested 2024-12-10T07:26:23,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583737 2024-12-10T07:26:23,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583737 2024-12-10T07:26:23,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583737) roll requested 2024-12-10T07:26:23,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583743 2024-12-10T07:26:23,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583743 2024-12-10T07:26:23,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583743) roll requested 2024-12-10T07:26:23,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583749 2024-12-10T07:26:23,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583749 2024-12-10T07:26:23,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583749) roll requested 2024-12-10T07:26:23,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583756 2024-12-10T07:26:23,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583756 2024-12-10T07:26:23,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583756) roll requested 2024-12-10T07:26:23,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583764 2024-12-10T07:26:23,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583764 2024-12-10T07:26:23,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583764) roll requested 2024-12-10T07:26:23,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583773 2024-12-10T07:26:23,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583773 2024-12-10T07:26:23,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583773) roll requested 2024-12-10T07:26:23,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583782 2024-12-10T07:26:23,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583782 2024-12-10T07:26:23,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583782) roll requested 2024-12-10T07:26:23,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583789 2024-12-10T07:26:23,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583789 2024-12-10T07:26:23,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583789) roll requested 2024-12-10T07:26:23,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583797 2024-12-10T07:26:23,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583797 2024-12-10T07:26:23,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583797) roll requested 2024-12-10T07:26:23,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583805 2024-12-10T07:26:23,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583805 2024-12-10T07:26:23,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583805) roll requested 2024-12-10T07:26:23,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583813 2024-12-10T07:26:23,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583813 2024-12-10T07:26:23,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583813) roll requested 2024-12-10T07:26:23,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583821 2024-12-10T07:26:23,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583821 2024-12-10T07:26:23,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583821) roll requested 2024-12-10T07:26:23,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583829 2024-12-10T07:26:23,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583829 2024-12-10T07:26:23,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583829) roll requested 2024-12-10T07:26:23,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583837 2024-12-10T07:26:23,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583837 2024-12-10T07:26:23,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583837) roll requested 2024-12-10T07:26:23,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583845 2024-12-10T07:26:23,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583845 2024-12-10T07:26:23,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583845) roll requested 2024-12-10T07:26:23,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583852 2024-12-10T07:26:23,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583852 2024-12-10T07:26:23,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583852) roll requested 2024-12-10T07:26:23,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583861 2024-12-10T07:26:23,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583861 2024-12-10T07:26:23,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583861) roll requested 2024-12-10T07:26:23,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583869 2024-12-10T07:26:23,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583869 2024-12-10T07:26:23,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583869) roll requested 2024-12-10T07:26:23,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583878 2024-12-10T07:26:23,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583878 2024-12-10T07:26:23,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583878) roll requested 2024-12-10T07:26:23,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583891 2024-12-10T07:26:23,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583891 2024-12-10T07:26:23,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583891) roll requested 2024-12-10T07:26:23,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583896 2024-12-10T07:26:23,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583896 2024-12-10T07:26:23,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583896) roll requested 2024-12-10T07:26:23,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583902 2024-12-10T07:26:23,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583902 2024-12-10T07:26:23,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583902) roll requested 2024-12-10T07:26:23,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583909 2024-12-10T07:26:23,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583909 2024-12-10T07:26:23,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583909) roll requested 2024-12-10T07:26:23,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583914 2024-12-10T07:26:23,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583914 2024-12-10T07:26:23,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583914) roll requested 2024-12-10T07:26:23,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583921 2024-12-10T07:26:23,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583921 2024-12-10T07:26:23,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583921) roll requested 2024-12-10T07:26:23,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583928 2024-12-10T07:26:23,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583928 2024-12-10T07:26:23,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583928) roll requested 2024-12-10T07:26:23,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583939 2024-12-10T07:26:23,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583939 2024-12-10T07:26:23,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583939) roll requested 2024-12-10T07:26:23,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583947 2024-12-10T07:26:23,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583947 2024-12-10T07:26:23,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583947) roll requested 2024-12-10T07:26:23,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583954 2024-12-10T07:26:23,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583954 2024-12-10T07:26:23,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583954) roll requested 2024-12-10T07:26:23,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583961 2024-12-10T07:26:23,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583961 2024-12-10T07:26:23,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583961) roll requested 2024-12-10T07:26:23,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583966 2024-12-10T07:26:23,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583966 2024-12-10T07:26:23,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583966) roll requested 2024-12-10T07:26:23,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583971 2024-12-10T07:26:23,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583971 2024-12-10T07:26:23,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583971) roll requested 2024-12-10T07:26:23,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583976 2024-12-10T07:26:23,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583976 2024-12-10T07:26:23,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583976) roll requested 2024-12-10T07:26:23,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583982 2024-12-10T07:26:23,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583982 2024-12-10T07:26:23,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583982) roll requested 2024-12-10T07:26:23,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583989 2024-12-10T07:26:23,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:23,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583989 2024-12-10T07:26:23,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:23,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:23,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583989) roll requested 2024-12-10T07:26:23,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815583997 2024-12-10T07:26:24,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583997 2024-12-10T07:26:24,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815583997) roll requested 2024-12-10T07:26:24,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584003 2024-12-10T07:26:24,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815583997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584003 2024-12-10T07:26:24,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584003) roll requested 2024-12-10T07:26:24,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584009 2024-12-10T07:26:24,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584009 2024-12-10T07:26:24,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584009) roll requested 2024-12-10T07:26:24,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584014 2024-12-10T07:26:24,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584014 2024-12-10T07:26:24,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584014) roll requested 2024-12-10T07:26:24,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584025 2024-12-10T07:26:24,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584025 2024-12-10T07:26:24,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584025) roll requested 2024-12-10T07:26:24,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584033 2024-12-10T07:26:24,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584033 2024-12-10T07:26:24,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584033) roll requested 2024-12-10T07:26:24,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584041 2024-12-10T07:26:24,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584041 2024-12-10T07:26:24,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584041) roll requested 2024-12-10T07:26:24,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584048 2024-12-10T07:26:24,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584048 2024-12-10T07:26:24,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584048) roll requested 2024-12-10T07:26:24,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584053 2024-12-10T07:26:24,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584053 2024-12-10T07:26:24,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584053) roll requested 2024-12-10T07:26:24,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584061 2024-12-10T07:26:24,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584061 2024-12-10T07:26:24,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584061) roll requested 2024-12-10T07:26:24,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584066 2024-12-10T07:26:24,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584066 2024-12-10T07:26:24,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584066) roll requested 2024-12-10T07:26:24,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584073 2024-12-10T07:26:24,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584073 2024-12-10T07:26:24,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584073) roll requested 2024-12-10T07:26:24,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584079 2024-12-10T07:26:24,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584079 2024-12-10T07:26:24,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584079) roll requested 2024-12-10T07:26:24,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584084 2024-12-10T07:26:24,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584084 2024-12-10T07:26:24,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584084) roll requested 2024-12-10T07:26:24,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584089 2024-12-10T07:26:24,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584089 2024-12-10T07:26:24,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584089) roll requested 2024-12-10T07:26:24,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584097 2024-12-10T07:26:24,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584097 2024-12-10T07:26:24,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584097) roll requested 2024-12-10T07:26:24,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584103 2024-12-10T07:26:24,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584103 2024-12-10T07:26:24,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584103) roll requested 2024-12-10T07:26:24,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584108 2024-12-10T07:26:24,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584108 2024-12-10T07:26:24,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584108) roll requested 2024-12-10T07:26:24,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584117 2024-12-10T07:26:24,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584117 2024-12-10T07:26:24,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584117) roll requested 2024-12-10T07:26:24,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584125 2024-12-10T07:26:24,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584125 2024-12-10T07:26:24,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584125) roll requested 2024-12-10T07:26:24,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584133 2024-12-10T07:26:24,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584133 2024-12-10T07:26:24,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584133) roll requested 2024-12-10T07:26:24,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584139 2024-12-10T07:26:24,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584139 2024-12-10T07:26:24,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584139) roll requested 2024-12-10T07:26:24,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584145 2024-12-10T07:26:24,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584145 2024-12-10T07:26:24,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584145) roll requested 2024-12-10T07:26:24,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584152 2024-12-10T07:26:24,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584152 2024-12-10T07:26:24,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584152) roll requested 2024-12-10T07:26:24,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584158 2024-12-10T07:26:24,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584158 2024-12-10T07:26:24,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584158) roll requested 2024-12-10T07:26:24,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584163 2024-12-10T07:26:24,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584163 2024-12-10T07:26:24,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584163) roll requested 2024-12-10T07:26:24,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584168 2024-12-10T07:26:24,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584168 2024-12-10T07:26:24,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584168) roll requested 2024-12-10T07:26:24,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584173 2024-12-10T07:26:24,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584173 2024-12-10T07:26:24,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584173) roll requested 2024-12-10T07:26:24,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584180 2024-12-10T07:26:24,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584180 2024-12-10T07:26:24,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584180) roll requested 2024-12-10T07:26:24,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584186 2024-12-10T07:26:24,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584186 2024-12-10T07:26:24,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584186) roll requested 2024-12-10T07:26:24,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584193 2024-12-10T07:26:24,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584193 2024-12-10T07:26:24,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584193) roll requested 2024-12-10T07:26:24,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584200 2024-12-10T07:26:24,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584200 2024-12-10T07:26:24,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584200) roll requested 2024-12-10T07:26:24,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584207 2024-12-10T07:26:24,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584207 2024-12-10T07:26:24,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584207) roll requested 2024-12-10T07:26:24,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584214 2024-12-10T07:26:24,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584214 2024-12-10T07:26:24,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584214) roll requested 2024-12-10T07:26:24,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584222 2024-12-10T07:26:24,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584222 2024-12-10T07:26:24,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584222) roll requested 2024-12-10T07:26:24,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584227 2024-12-10T07:26:24,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584227 2024-12-10T07:26:24,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584227) roll requested 2024-12-10T07:26:24,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584232 2024-12-10T07:26:24,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584232 2024-12-10T07:26:24,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584232) roll requested 2024-12-10T07:26:24,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584238 2024-12-10T07:26:24,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584238 2024-12-10T07:26:24,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584238) roll requested 2024-12-10T07:26:24,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584245 2024-12-10T07:26:24,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584245 2024-12-10T07:26:24,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584245) roll requested 2024-12-10T07:26:24,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584252 2024-12-10T07:26:24,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584252 2024-12-10T07:26:24,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584252) roll requested 2024-12-10T07:26:24,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584260 2024-12-10T07:26:24,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584260 2024-12-10T07:26:24,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584260) roll requested 2024-12-10T07:26:24,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584268 2024-12-10T07:26:24,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584268 2024-12-10T07:26:24,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584268) roll requested 2024-12-10T07:26:24,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584277 2024-12-10T07:26:24,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584277 2024-12-10T07:26:24,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584277) roll requested 2024-12-10T07:26:24,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584285 2024-12-10T07:26:24,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584285 2024-12-10T07:26:24,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584285) roll requested 2024-12-10T07:26:24,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584292 2024-12-10T07:26:24,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584292 2024-12-10T07:26:24,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584292) roll requested 2024-12-10T07:26:24,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584299 2024-12-10T07:26:24,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584299 2024-12-10T07:26:24,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584299) roll requested 2024-12-10T07:26:24,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584308 2024-12-10T07:26:24,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584308 2024-12-10T07:26:24,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584308) roll requested 2024-12-10T07:26:24,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584315 2024-12-10T07:26:24,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584315 2024-12-10T07:26:24,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584315) roll requested 2024-12-10T07:26:24,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584320 2024-12-10T07:26:24,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584320 2024-12-10T07:26:24,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584320) roll requested 2024-12-10T07:26:24,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584325 2024-12-10T07:26:24,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584325 2024-12-10T07:26:24,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584325) roll requested 2024-12-10T07:26:24,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584331 2024-12-10T07:26:24,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584331 2024-12-10T07:26:24,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584331) roll requested 2024-12-10T07:26:24,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584339 2024-12-10T07:26:24,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584339 2024-12-10T07:26:24,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584339) roll requested 2024-12-10T07:26:24,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584347 2024-12-10T07:26:24,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584347 2024-12-10T07:26:24,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584347) roll requested 2024-12-10T07:26:24,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584355 2024-12-10T07:26:24,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584355 2024-12-10T07:26:24,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584355) roll requested 2024-12-10T07:26:24,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584363 2024-12-10T07:26:24,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584363 2024-12-10T07:26:24,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584363) roll requested 2024-12-10T07:26:24,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584373 2024-12-10T07:26:24,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584373 2024-12-10T07:26:24,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584373) roll requested 2024-12-10T07:26:24,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584380 2024-12-10T07:26:24,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584380 2024-12-10T07:26:24,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584380) roll requested 2024-12-10T07:26:24,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584386 2024-12-10T07:26:24,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584386 2024-12-10T07:26:24,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584386) roll requested 2024-12-10T07:26:24,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584394 2024-12-10T07:26:24,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584394 2024-12-10T07:26:24,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584394) roll requested 2024-12-10T07:26:24,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584403 2024-12-10T07:26:24,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584403 2024-12-10T07:26:24,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584403) roll requested 2024-12-10T07:26:24,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584412 2024-12-10T07:26:24,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584412 2024-12-10T07:26:24,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584412) roll requested 2024-12-10T07:26:24,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584420 2024-12-10T07:26:24,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584420 2024-12-10T07:26:24,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584420) roll requested 2024-12-10T07:26:24,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584427 2024-12-10T07:26:24,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584427 2024-12-10T07:26:24,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584427) roll requested 2024-12-10T07:26:24,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584434 2024-12-10T07:26:24,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584434 2024-12-10T07:26:24,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584434) roll requested 2024-12-10T07:26:24,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584442 2024-12-10T07:26:24,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584442 2024-12-10T07:26:24,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584442) roll requested 2024-12-10T07:26:24,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584451 2024-12-10T07:26:24,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584451 2024-12-10T07:26:24,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584451) roll requested 2024-12-10T07:26:24,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584459 2024-12-10T07:26:24,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584459 2024-12-10T07:26:24,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584459) roll requested 2024-12-10T07:26:24,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584466 2024-12-10T07:26:24,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584466 2024-12-10T07:26:24,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584466) roll requested 2024-12-10T07:26:24,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584473 2024-12-10T07:26:24,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584473 2024-12-10T07:26:24,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584473) roll requested 2024-12-10T07:26:24,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584482 2024-12-10T07:26:24,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584482 2024-12-10T07:26:24,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584482) roll requested 2024-12-10T07:26:24,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584487 2024-12-10T07:26:24,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584487 2024-12-10T07:26:24,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584487) roll requested 2024-12-10T07:26:24,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584493 2024-12-10T07:26:24,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584493 2024-12-10T07:26:24,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584493) roll requested 2024-12-10T07:26:24,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584499 2024-12-10T07:26:24,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584499 2024-12-10T07:26:24,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584499) roll requested 2024-12-10T07:26:24,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584506 2024-12-10T07:26:24,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584506 2024-12-10T07:26:24,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584506) roll requested 2024-12-10T07:26:24,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584518 2024-12-10T07:26:24,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584518 2024-12-10T07:26:24,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584518) roll requested 2024-12-10T07:26:24,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584526 2024-12-10T07:26:24,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584526 2024-12-10T07:26:24,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584526) roll requested 2024-12-10T07:26:24,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584534 2024-12-10T07:26:24,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584534 2024-12-10T07:26:24,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584534) roll requested 2024-12-10T07:26:24,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584541 2024-12-10T07:26:24,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584541 2024-12-10T07:26:24,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584541) roll requested 2024-12-10T07:26:24,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584548 2024-12-10T07:26:24,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584548 2024-12-10T07:26:24,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584548) roll requested 2024-12-10T07:26:24,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584557 2024-12-10T07:26:24,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584557 2024-12-10T07:26:24,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584557) roll requested 2024-12-10T07:26:24,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584565 2024-12-10T07:26:24,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584565 2024-12-10T07:26:24,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584565) roll requested 2024-12-10T07:26:24,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584573 2024-12-10T07:26:24,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584573 2024-12-10T07:26:24,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584573) roll requested 2024-12-10T07:26:24,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584581 2024-12-10T07:26:24,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584581 2024-12-10T07:26:24,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584581) roll requested 2024-12-10T07:26:24,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584588 2024-12-10T07:26:24,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584588 2024-12-10T07:26:24,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584588) roll requested 2024-12-10T07:26:24,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584595 2024-12-10T07:26:24,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584595 2024-12-10T07:26:24,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584595) roll requested 2024-12-10T07:26:24,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584604 2024-12-10T07:26:24,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584604 2024-12-10T07:26:24,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584604) roll requested 2024-12-10T07:26:24,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584612 2024-12-10T07:26:24,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584612 2024-12-10T07:26:24,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584612) roll requested 2024-12-10T07:26:24,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584619 2024-12-10T07:26:24,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584619 2024-12-10T07:26:24,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584619) roll requested 2024-12-10T07:26:24,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584626 2024-12-10T07:26:24,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584626 2024-12-10T07:26:24,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584626) roll requested 2024-12-10T07:26:24,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584635 2024-12-10T07:26:24,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584635 2024-12-10T07:26:24,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584635) roll requested 2024-12-10T07:26:24,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584642 2024-12-10T07:26:24,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584642 2024-12-10T07:26:24,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584642) roll requested 2024-12-10T07:26:24,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584650 2024-12-10T07:26:24,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584650 2024-12-10T07:26:24,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584650) roll requested 2024-12-10T07:26:24,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584659 2024-12-10T07:26:24,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584659 2024-12-10T07:26:24,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584659) roll requested 2024-12-10T07:26:24,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584665 2024-12-10T07:26:24,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584665 2024-12-10T07:26:24,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584665) roll requested 2024-12-10T07:26:24,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584671 2024-12-10T07:26:24,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584671 2024-12-10T07:26:24,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584671) roll requested 2024-12-10T07:26:24,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584678 2024-12-10T07:26:24,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584678 2024-12-10T07:26:24,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584678) roll requested 2024-12-10T07:26:24,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584687 2024-12-10T07:26:24,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584687 2024-12-10T07:26:24,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584687) roll requested 2024-12-10T07:26:24,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584696 2024-12-10T07:26:24,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584696 2024-12-10T07:26:24,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584696) roll requested 2024-12-10T07:26:24,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584706 2024-12-10T07:26:24,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584706 2024-12-10T07:26:24,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584706) roll requested 2024-12-10T07:26:24,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584716 2024-12-10T07:26:24,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584716 2024-12-10T07:26:24,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584716) roll requested 2024-12-10T07:26:24,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584725 2024-12-10T07:26:24,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584725 2024-12-10T07:26:24,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584725) roll requested 2024-12-10T07:26:24,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584732 2024-12-10T07:26:24,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584732 2024-12-10T07:26:24,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584732) roll requested 2024-12-10T07:26:24,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584741 2024-12-10T07:26:24,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584741 2024-12-10T07:26:24,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584741) roll requested 2024-12-10T07:26:24,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584749 2024-12-10T07:26:24,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584749 2024-12-10T07:26:24,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584749) roll requested 2024-12-10T07:26:24,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584755 2024-12-10T07:26:24,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584755 2024-12-10T07:26:24,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584755) roll requested 2024-12-10T07:26:24,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584763 2024-12-10T07:26:24,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584763 2024-12-10T07:26:24,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584763) roll requested 2024-12-10T07:26:24,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584770 2024-12-10T07:26:24,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584770 2024-12-10T07:26:24,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584770) roll requested 2024-12-10T07:26:24,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584778 2024-12-10T07:26:24,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584778 2024-12-10T07:26:24,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584778) roll requested 2024-12-10T07:26:24,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584787 2024-12-10T07:26:24,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584787 2024-12-10T07:26:24,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584787) roll requested 2024-12-10T07:26:24,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584794 2024-12-10T07:26:24,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584794 2024-12-10T07:26:24,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584794) roll requested 2024-12-10T07:26:24,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584801 2024-12-10T07:26:24,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584801 2024-12-10T07:26:24,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584801) roll requested 2024-12-10T07:26:24,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584809 2024-12-10T07:26:24,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584809 2024-12-10T07:26:24,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584809) roll requested 2024-12-10T07:26:24,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584818 2024-12-10T07:26:24,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584818 2024-12-10T07:26:24,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584818) roll requested 2024-12-10T07:26:24,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584826 2024-12-10T07:26:24,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584826 2024-12-10T07:26:24,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584826) roll requested 2024-12-10T07:26:24,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584835 2024-12-10T07:26:24,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584835 2024-12-10T07:26:24,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584835) roll requested 2024-12-10T07:26:24,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584844 2024-12-10T07:26:24,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584844 2024-12-10T07:26:24,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584844) roll requested 2024-12-10T07:26:24,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584851 2024-12-10T07:26:24,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584851 2024-12-10T07:26:24,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584851) roll requested 2024-12-10T07:26:24,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584861 2024-12-10T07:26:24,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584861 2024-12-10T07:26:24,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584861) roll requested 2024-12-10T07:26:24,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584869 2024-12-10T07:26:24,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584869 2024-12-10T07:26:24,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584869) roll requested 2024-12-10T07:26:24,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584877 2024-12-10T07:26:24,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584877 2024-12-10T07:26:24,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584877) roll requested 2024-12-10T07:26:24,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584885 2024-12-10T07:26:24,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584885 2024-12-10T07:26:24,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584885) roll requested 2024-12-10T07:26:24,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584893 2024-12-10T07:26:24,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584893 2024-12-10T07:26:24,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584893) roll requested 2024-12-10T07:26:24,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584901 2024-12-10T07:26:24,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584901 2024-12-10T07:26:24,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584901) roll requested 2024-12-10T07:26:24,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584911 2024-12-10T07:26:24,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584911 2024-12-10T07:26:24,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584911) roll requested 2024-12-10T07:26:24,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584920 2024-12-10T07:26:24,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584920 2024-12-10T07:26:24,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584920) roll requested 2024-12-10T07:26:24,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584926 2024-12-10T07:26:24,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584926 2024-12-10T07:26:24,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584926) roll requested 2024-12-10T07:26:24,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584934 2024-12-10T07:26:24,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584934 2024-12-10T07:26:24,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584934) roll requested 2024-12-10T07:26:24,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584941 2024-12-10T07:26:24,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584941 2024-12-10T07:26:24,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584941) roll requested 2024-12-10T07:26:24,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584948 2024-12-10T07:26:24,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584948 2024-12-10T07:26:24,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584948) roll requested 2024-12-10T07:26:24,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584956 2024-12-10T07:26:24,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584956 2024-12-10T07:26:24,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584956) roll requested 2024-12-10T07:26:24,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584965 2024-12-10T07:26:24,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584965 2024-12-10T07:26:24,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584965) roll requested 2024-12-10T07:26:24,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584972 2024-12-10T07:26:24,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584972 2024-12-10T07:26:24,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584972) roll requested 2024-12-10T07:26:24,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584981 2024-12-10T07:26:24,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584981 2024-12-10T07:26:24,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584981) roll requested 2024-12-10T07:26:24,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584989 2024-12-10T07:26:24,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:24,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584989 2024-12-10T07:26:24,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:24,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:24,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584989) roll requested 2024-12-10T07:26:24,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815584996 2024-12-10T07:26:25,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584996 2024-12-10T07:26:25,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815584996) roll requested 2024-12-10T07:26:25,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585003 2024-12-10T07:26:25,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815584996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585003 2024-12-10T07:26:25,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585003) roll requested 2024-12-10T07:26:25,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585011 2024-12-10T07:26:25,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585011 2024-12-10T07:26:25,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585011) roll requested 2024-12-10T07:26:25,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585019 2024-12-10T07:26:25,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585019 2024-12-10T07:26:25,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585019) roll requested 2024-12-10T07:26:25,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585026 2024-12-10T07:26:25,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585026 2024-12-10T07:26:25,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585026) roll requested 2024-12-10T07:26:25,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585034 2024-12-10T07:26:25,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585034 2024-12-10T07:26:25,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585034) roll requested 2024-12-10T07:26:25,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585044 2024-12-10T07:26:25,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585044 2024-12-10T07:26:25,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585044) roll requested 2024-12-10T07:26:25,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585052 2024-12-10T07:26:25,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585052 2024-12-10T07:26:25,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585052) roll requested 2024-12-10T07:26:25,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585062 2024-12-10T07:26:25,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585062 2024-12-10T07:26:25,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585062) roll requested 2024-12-10T07:26:25,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585071 2024-12-10T07:26:25,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585071 2024-12-10T07:26:25,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585071) roll requested 2024-12-10T07:26:25,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585079 2024-12-10T07:26:25,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585079 2024-12-10T07:26:25,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585079) roll requested 2024-12-10T07:26:25,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585087 2024-12-10T07:26:25,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585087 2024-12-10T07:26:25,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585087) roll requested 2024-12-10T07:26:25,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585156 2024-12-10T07:26:25,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585156 2024-12-10T07:26:25,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585156) roll requested 2024-12-10T07:26:25,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585162 2024-12-10T07:26:25,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585162 2024-12-10T07:26:25,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585162) roll requested 2024-12-10T07:26:25,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585170 2024-12-10T07:26:25,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585170 2024-12-10T07:26:25,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585170) roll requested 2024-12-10T07:26:25,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585177 2024-12-10T07:26:25,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585177 2024-12-10T07:26:25,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585177) roll requested 2024-12-10T07:26:25,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585185 2024-12-10T07:26:25,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585185 2024-12-10T07:26:25,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585185) roll requested 2024-12-10T07:26:25,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585194 2024-12-10T07:26:25,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585194 2024-12-10T07:26:25,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585194) roll requested 2024-12-10T07:26:25,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585203 2024-12-10T07:26:25,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585203 2024-12-10T07:26:25,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585203) roll requested 2024-12-10T07:26:25,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585212 2024-12-10T07:26:25,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585212 2024-12-10T07:26:25,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585212) roll requested 2024-12-10T07:26:25,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585222 2024-12-10T07:26:25,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585222 2024-12-10T07:26:25,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585222) roll requested 2024-12-10T07:26:25,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585230 2024-12-10T07:26:25,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585230 2024-12-10T07:26:25,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585230) roll requested 2024-12-10T07:26:25,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585238 2024-12-10T07:26:25,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585238 2024-12-10T07:26:25,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585238) roll requested 2024-12-10T07:26:25,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585246 2024-12-10T07:26:25,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585246 2024-12-10T07:26:25,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585246) roll requested 2024-12-10T07:26:25,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585254 2024-12-10T07:26:25,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585254 2024-12-10T07:26:25,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585254) roll requested 2024-12-10T07:26:25,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585264 2024-12-10T07:26:25,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585264 2024-12-10T07:26:25,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585264) roll requested 2024-12-10T07:26:25,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585271 2024-12-10T07:26:25,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585271 2024-12-10T07:26:25,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585271) roll requested 2024-12-10T07:26:25,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585278 2024-12-10T07:26:25,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585278 2024-12-10T07:26:25,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585278) roll requested 2024-12-10T07:26:25,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585285 2024-12-10T07:26:25,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585285 2024-12-10T07:26:25,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585285) roll requested 2024-12-10T07:26:25,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585290 2024-12-10T07:26:25,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585290 2024-12-10T07:26:25,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585290) roll requested 2024-12-10T07:26:25,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585299 2024-12-10T07:26:25,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585299 2024-12-10T07:26:25,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585299) roll requested 2024-12-10T07:26:25,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585306 2024-12-10T07:26:25,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585306 2024-12-10T07:26:25,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585306) roll requested 2024-12-10T07:26:25,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585314 2024-12-10T07:26:25,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585314 2024-12-10T07:26:25,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585314) roll requested 2024-12-10T07:26:25,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585322 2024-12-10T07:26:25,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585322 2024-12-10T07:26:25,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585322) roll requested 2024-12-10T07:26:25,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585330 2024-12-10T07:26:25,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585330 2024-12-10T07:26:25,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585330) roll requested 2024-12-10T07:26:25,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585337 2024-12-10T07:26:25,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585337 2024-12-10T07:26:25,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585337) roll requested 2024-12-10T07:26:25,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585346 2024-12-10T07:26:25,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585346 2024-12-10T07:26:25,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585346) roll requested 2024-12-10T07:26:25,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585355 2024-12-10T07:26:25,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585355 2024-12-10T07:26:25,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585355) roll requested 2024-12-10T07:26:25,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585362 2024-12-10T07:26:25,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585362 2024-12-10T07:26:25,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585362) roll requested 2024-12-10T07:26:25,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585369 2024-12-10T07:26:25,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585369 2024-12-10T07:26:25,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585369) roll requested 2024-12-10T07:26:25,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585376 2024-12-10T07:26:25,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585376 2024-12-10T07:26:25,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585376) roll requested 2024-12-10T07:26:25,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585382 2024-12-10T07:26:25,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585382 2024-12-10T07:26:25,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585382) roll requested 2024-12-10T07:26:25,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585390 2024-12-10T07:26:25,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585390 2024-12-10T07:26:25,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585390) roll requested 2024-12-10T07:26:25,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585397 2024-12-10T07:26:25,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585397 2024-12-10T07:26:25,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585397) roll requested 2024-12-10T07:26:25,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585406 2024-12-10T07:26:25,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585406 2024-12-10T07:26:25,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585406) roll requested 2024-12-10T07:26:25,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585415 2024-12-10T07:26:25,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585415 2024-12-10T07:26:25,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585415) roll requested 2024-12-10T07:26:25,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585424 2024-12-10T07:26:25,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585424 2024-12-10T07:26:25,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585424) roll requested 2024-12-10T07:26:25,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585433 2024-12-10T07:26:25,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585433 2024-12-10T07:26:25,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585433) roll requested 2024-12-10T07:26:25,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585442 2024-12-10T07:26:25,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585442 2024-12-10T07:26:25,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585442) roll requested 2024-12-10T07:26:25,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585448 2024-12-10T07:26:25,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585448 2024-12-10T07:26:25,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585448) roll requested 2024-12-10T07:26:25,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585455 2024-12-10T07:26:25,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585455 2024-12-10T07:26:25,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585455) roll requested 2024-12-10T07:26:25,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585461 2024-12-10T07:26:25,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585461 2024-12-10T07:26:25,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585461) roll requested 2024-12-10T07:26:25,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585470 2024-12-10T07:26:25,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585470 2024-12-10T07:26:25,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585470) roll requested 2024-12-10T07:26:25,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585478 2024-12-10T07:26:25,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585478 2024-12-10T07:26:25,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585478) roll requested 2024-12-10T07:26:25,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585482 2024-12-10T07:26:25,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585482 2024-12-10T07:26:25,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585482) roll requested 2024-12-10T07:26:25,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585488 2024-12-10T07:26:25,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585488 2024-12-10T07:26:25,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585488) roll requested 2024-12-10T07:26:25,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585494 2024-12-10T07:26:25,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585494 2024-12-10T07:26:25,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585494) roll requested 2024-12-10T07:26:25,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585501 2024-12-10T07:26:25,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585501 2024-12-10T07:26:25,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585501) roll requested 2024-12-10T07:26:25,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585508 2024-12-10T07:26:25,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585508 2024-12-10T07:26:25,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585508) roll requested 2024-12-10T07:26:25,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585514 2024-12-10T07:26:25,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585514 2024-12-10T07:26:25,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585514) roll requested 2024-12-10T07:26:25,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585521 2024-12-10T07:26:25,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585521 2024-12-10T07:26:25,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585521) roll requested 2024-12-10T07:26:25,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585529 2024-12-10T07:26:25,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585529 2024-12-10T07:26:25,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585529) roll requested 2024-12-10T07:26:25,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585537 2024-12-10T07:26:25,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585537 2024-12-10T07:26:25,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585537) roll requested 2024-12-10T07:26:25,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585545 2024-12-10T07:26:25,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585545 2024-12-10T07:26:25,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585545) roll requested 2024-12-10T07:26:25,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585554 2024-12-10T07:26:25,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585554 2024-12-10T07:26:25,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585554) roll requested 2024-12-10T07:26:25,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585562 2024-12-10T07:26:25,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585562 2024-12-10T07:26:25,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585562) roll requested 2024-12-10T07:26:25,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585572 2024-12-10T07:26:25,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585572 2024-12-10T07:26:25,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585572) roll requested 2024-12-10T07:26:25,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585581 2024-12-10T07:26:25,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585581 2024-12-10T07:26:25,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585581) roll requested 2024-12-10T07:26:25,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585588 2024-12-10T07:26:25,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585588 2024-12-10T07:26:25,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585588) roll requested 2024-12-10T07:26:25,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585596 2024-12-10T07:26:25,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585596 2024-12-10T07:26:25,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585596) roll requested 2024-12-10T07:26:25,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585604 2024-12-10T07:26:25,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585604 2024-12-10T07:26:25,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585604) roll requested 2024-12-10T07:26:25,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585615 2024-12-10T07:26:25,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585615 2024-12-10T07:26:25,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585615) roll requested 2024-12-10T07:26:25,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585621 2024-12-10T07:26:25,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585621 2024-12-10T07:26:25,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585621) roll requested 2024-12-10T07:26:25,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585627 2024-12-10T07:26:25,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585627 2024-12-10T07:26:25,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585627) roll requested 2024-12-10T07:26:25,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585636 2024-12-10T07:26:25,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585636 2024-12-10T07:26:25,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585636) roll requested 2024-12-10T07:26:25,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585645 2024-12-10T07:26:25,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585645 2024-12-10T07:26:25,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585645) roll requested 2024-12-10T07:26:25,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585653 2024-12-10T07:26:25,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585653 2024-12-10T07:26:25,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585653) roll requested 2024-12-10T07:26:25,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585661 2024-12-10T07:26:25,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585661 2024-12-10T07:26:25,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585661) roll requested 2024-12-10T07:26:25,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585669 2024-12-10T07:26:25,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585669 2024-12-10T07:26:25,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585669) roll requested 2024-12-10T07:26:25,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585674 2024-12-10T07:26:25,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585674 2024-12-10T07:26:25,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585674) roll requested 2024-12-10T07:26:25,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585680 2024-12-10T07:26:25,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585680 2024-12-10T07:26:25,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585680) roll requested 2024-12-10T07:26:25,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585685 2024-12-10T07:26:25,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585685 2024-12-10T07:26:25,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585685) roll requested 2024-12-10T07:26:25,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585691 2024-12-10T07:26:25,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585691 2024-12-10T07:26:25,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585691) roll requested 2024-12-10T07:26:25,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585697 2024-12-10T07:26:25,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585697 2024-12-10T07:26:25,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585697) roll requested 2024-12-10T07:26:25,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585702 2024-12-10T07:26:25,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585702 2024-12-10T07:26:25,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585702) roll requested 2024-12-10T07:26:25,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585708 2024-12-10T07:26:25,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585708 2024-12-10T07:26:25,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585708) roll requested 2024-12-10T07:26:25,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585713 2024-12-10T07:26:25,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585713 2024-12-10T07:26:25,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585713) roll requested 2024-12-10T07:26:25,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585718 2024-12-10T07:26:25,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585718 2024-12-10T07:26:25,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585718) roll requested 2024-12-10T07:26:25,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585723 2024-12-10T07:26:25,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585723 2024-12-10T07:26:25,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585723) roll requested 2024-12-10T07:26:25,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585727 2024-12-10T07:26:25,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585727 2024-12-10T07:26:25,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585727) roll requested 2024-12-10T07:26:25,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585733 2024-12-10T07:26:25,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585733 2024-12-10T07:26:25,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585733) roll requested 2024-12-10T07:26:25,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585740 2024-12-10T07:26:25,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585740 2024-12-10T07:26:25,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585740) roll requested 2024-12-10T07:26:25,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585746 2024-12-10T07:26:25,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585746 2024-12-10T07:26:25,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585746) roll requested 2024-12-10T07:26:25,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585753 2024-12-10T07:26:25,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585753 2024-12-10T07:26:25,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585753) roll requested 2024-12-10T07:26:25,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585760 2024-12-10T07:26:25,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585760 2024-12-10T07:26:25,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585760) roll requested 2024-12-10T07:26:25,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585768 2024-12-10T07:26:25,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585768 2024-12-10T07:26:25,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585768) roll requested 2024-12-10T07:26:25,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585777 2024-12-10T07:26:25,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585777 2024-12-10T07:26:25,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585777) roll requested 2024-12-10T07:26:25,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585784 2024-12-10T07:26:25,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585784 2024-12-10T07:26:25,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585784) roll requested 2024-12-10T07:26:25,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585792 2024-12-10T07:26:25,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585792 2024-12-10T07:26:25,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585792) roll requested 2024-12-10T07:26:25,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585799 2024-12-10T07:26:25,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585799 2024-12-10T07:26:25,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585799) roll requested 2024-12-10T07:26:25,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585807 2024-12-10T07:26:25,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585807 2024-12-10T07:26:25,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585807) roll requested 2024-12-10T07:26:25,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585817 2024-12-10T07:26:25,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585817 2024-12-10T07:26:25,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585817) roll requested 2024-12-10T07:26:25,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585826 2024-12-10T07:26:25,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585826 2024-12-10T07:26:25,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585826) roll requested 2024-12-10T07:26:25,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585831 2024-12-10T07:26:25,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585831 2024-12-10T07:26:25,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585831) roll requested 2024-12-10T07:26:25,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585836 2024-12-10T07:26:25,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585836 2024-12-10T07:26:25,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585836) roll requested 2024-12-10T07:26:25,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585844 2024-12-10T07:26:25,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585844 2024-12-10T07:26:25,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585844) roll requested 2024-12-10T07:26:25,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585851 2024-12-10T07:26:25,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585851 2024-12-10T07:26:25,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585851) roll requested 2024-12-10T07:26:25,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585859 2024-12-10T07:26:25,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585859 2024-12-10T07:26:25,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585859) roll requested 2024-12-10T07:26:25,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585868 2024-12-10T07:26:25,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585868 2024-12-10T07:26:25,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585868) roll requested 2024-12-10T07:26:25,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585877 2024-12-10T07:26:25,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585877 2024-12-10T07:26:25,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585877) roll requested 2024-12-10T07:26:25,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585883 2024-12-10T07:26:25,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585883 2024-12-10T07:26:25,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585883) roll requested 2024-12-10T07:26:25,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585891 2024-12-10T07:26:25,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585891 2024-12-10T07:26:25,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585891) roll requested 2024-12-10T07:26:25,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585899 2024-12-10T07:26:25,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585899 2024-12-10T07:26:25,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585899) roll requested 2024-12-10T07:26:25,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585906 2024-12-10T07:26:25,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585906 2024-12-10T07:26:25,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585906) roll requested 2024-12-10T07:26:25,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585914 2024-12-10T07:26:25,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585914 2024-12-10T07:26:25,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585914) roll requested 2024-12-10T07:26:25,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585922 2024-12-10T07:26:25,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585922 2024-12-10T07:26:25,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585922) roll requested 2024-12-10T07:26:25,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585929 2024-12-10T07:26:25,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585929 2024-12-10T07:26:25,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585929) roll requested 2024-12-10T07:26:25,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585938 2024-12-10T07:26:25,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585938 2024-12-10T07:26:25,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585938) roll requested 2024-12-10T07:26:25,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585945 2024-12-10T07:26:25,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585945 2024-12-10T07:26:25,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585945) roll requested 2024-12-10T07:26:25,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585951 2024-12-10T07:26:25,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585951 2024-12-10T07:26:25,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585951) roll requested 2024-12-10T07:26:25,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585959 2024-12-10T07:26:25,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585959 2024-12-10T07:26:25,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585959) roll requested 2024-12-10T07:26:25,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585968 2024-12-10T07:26:25,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585968 2024-12-10T07:26:25,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585968) roll requested 2024-12-10T07:26:25,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585975 2024-12-10T07:26:25,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585975 2024-12-10T07:26:25,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585975) roll requested 2024-12-10T07:26:25,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585981 2024-12-10T07:26:25,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585981 2024-12-10T07:26:25,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585981) roll requested 2024-12-10T07:26:25,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585986 2024-12-10T07:26:25,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585986 2024-12-10T07:26:25,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585986) roll requested 2024-12-10T07:26:25,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585990 2024-12-10T07:26:25,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:25,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585990 2024-12-10T07:26:25,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:25,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:25,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585990) roll requested 2024-12-10T07:26:25,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815585996 2024-12-10T07:26:26,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585996 2024-12-10T07:26:26,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815585996) roll requested 2024-12-10T07:26:26,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586005 2024-12-10T07:26:26,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815585996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586005 2024-12-10T07:26:26,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586005) roll requested 2024-12-10T07:26:26,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586014 2024-12-10T07:26:26,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586014 2024-12-10T07:26:26,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586014) roll requested 2024-12-10T07:26:26,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586021 2024-12-10T07:26:26,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586021 2024-12-10T07:26:26,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586021) roll requested 2024-12-10T07:26:26,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586029 2024-12-10T07:26:26,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586029 2024-12-10T07:26:26,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586021 is not closed yet, will try archiving it next time 2024-12-10T07:26:26,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586029) roll requested 2024-12-10T07:26:26,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586034 2024-12-10T07:26:26,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586034 2024-12-10T07:26:26,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586034) roll requested 2024-12-10T07:26:26,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586039 2024-12-10T07:26:26,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586039 2024-12-10T07:26:26,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586039) roll requested 2024-12-10T07:26:26,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586044 2024-12-10T07:26:26,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586044 2024-12-10T07:26:26,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586044) roll requested 2024-12-10T07:26:26,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586052 2024-12-10T07:26:26,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586052 2024-12-10T07:26:26,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586052) roll requested 2024-12-10T07:26:26,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586058 2024-12-10T07:26:26,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586058 2024-12-10T07:26:26,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586058) roll requested 2024-12-10T07:26:26,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586064 2024-12-10T07:26:26,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586064 2024-12-10T07:26:26,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586064) roll requested 2024-12-10T07:26:26,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586071 2024-12-10T07:26:26,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586071 2024-12-10T07:26:26,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586071) roll requested 2024-12-10T07:26:26,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586077 2024-12-10T07:26:26,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586077 2024-12-10T07:26:26,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586077) roll requested 2024-12-10T07:26:26,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586085 2024-12-10T07:26:26,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586085 2024-12-10T07:26:26,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586085) roll requested 2024-12-10T07:26:26,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586092 2024-12-10T07:26:26,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586092 2024-12-10T07:26:26,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586092) roll requested 2024-12-10T07:26:26,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586099 2024-12-10T07:26:26,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586099 2024-12-10T07:26:26,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586099) roll requested 2024-12-10T07:26:26,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586106 2024-12-10T07:26:26,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586106 2024-12-10T07:26:26,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586106) roll requested 2024-12-10T07:26:26,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586115 2024-12-10T07:26:26,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586115 2024-12-10T07:26:26,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586115) roll requested 2024-12-10T07:26:26,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586120 2024-12-10T07:26:26,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586120 2024-12-10T07:26:26,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586120) roll requested 2024-12-10T07:26:26,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586125 2024-12-10T07:26:26,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586125 2024-12-10T07:26:26,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586125) roll requested 2024-12-10T07:26:26,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586131 2024-12-10T07:26:26,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586131 2024-12-10T07:26:26,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586131) roll requested 2024-12-10T07:26:26,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586136 2024-12-10T07:26:26,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586136 2024-12-10T07:26:26,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586136) roll requested 2024-12-10T07:26:26,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586142 2024-12-10T07:26:26,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586142 2024-12-10T07:26:26,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586142) roll requested 2024-12-10T07:26:26,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586148 2024-12-10T07:26:26,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586148 2024-12-10T07:26:26,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586148) roll requested 2024-12-10T07:26:26,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586157 2024-12-10T07:26:26,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586157 2024-12-10T07:26:26,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586157) roll requested 2024-12-10T07:26:26,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586165 2024-12-10T07:26:26,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586165 2024-12-10T07:26:26,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586165) roll requested 2024-12-10T07:26:26,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586172 2024-12-10T07:26:26,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586172 2024-12-10T07:26:26,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586172) roll requested 2024-12-10T07:26:26,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586180 2024-12-10T07:26:26,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586180 2024-12-10T07:26:26,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586180) roll requested 2024-12-10T07:26:26,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586188 2024-12-10T07:26:26,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586188 2024-12-10T07:26:26,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586188) roll requested 2024-12-10T07:26:26,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586197 2024-12-10T07:26:26,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586197 2024-12-10T07:26:26,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586197) roll requested 2024-12-10T07:26:26,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586207 2024-12-10T07:26:26,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586207 2024-12-10T07:26:26,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586207) roll requested 2024-12-10T07:26:26,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586216 2024-12-10T07:26:26,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586216 2024-12-10T07:26:26,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586216) roll requested 2024-12-10T07:26:26,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586223 2024-12-10T07:26:26,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586223 2024-12-10T07:26:26,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586223) roll requested 2024-12-10T07:26:26,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586232 2024-12-10T07:26:26,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586232 2024-12-10T07:26:26,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586232) roll requested 2024-12-10T07:26:26,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586240 2024-12-10T07:26:26,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586240 2024-12-10T07:26:26,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586240) roll requested 2024-12-10T07:26:26,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586249 2024-12-10T07:26:26,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586249 2024-12-10T07:26:26,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586249) roll requested 2024-12-10T07:26:26,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586258 2024-12-10T07:26:26,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586258 2024-12-10T07:26:26,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586258) roll requested 2024-12-10T07:26:26,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586268 2024-12-10T07:26:26,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586268 2024-12-10T07:26:26,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586268) roll requested 2024-12-10T07:26:26,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586280 2024-12-10T07:26:26,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586280 2024-12-10T07:26:26,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586280) roll requested 2024-12-10T07:26:26,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586289 2024-12-10T07:26:26,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586289 2024-12-10T07:26:26,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586289) roll requested 2024-12-10T07:26:26,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586301 2024-12-10T07:26:26,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586301 2024-12-10T07:26:26,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586301) roll requested 2024-12-10T07:26:26,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586306 2024-12-10T07:26:26,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586306 2024-12-10T07:26:26,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586306) roll requested 2024-12-10T07:26:26,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586311 2024-12-10T07:26:26,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586311 2024-12-10T07:26:26,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586311) roll requested 2024-12-10T07:26:26,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586318 2024-12-10T07:26:26,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586318 2024-12-10T07:26:26,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586318) roll requested 2024-12-10T07:26:26,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586325 2024-12-10T07:26:26,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586325 2024-12-10T07:26:26,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586325) roll requested 2024-12-10T07:26:26,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586332 2024-12-10T07:26:26,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586332 2024-12-10T07:26:26,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586332) roll requested 2024-12-10T07:26:26,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586339 2024-12-10T07:26:26,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586339 2024-12-10T07:26:26,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586339) roll requested 2024-12-10T07:26:26,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586349 2024-12-10T07:26:26,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586349 2024-12-10T07:26:26,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586349) roll requested 2024-12-10T07:26:26,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586357 2024-12-10T07:26:26,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586357 2024-12-10T07:26:26,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586357) roll requested 2024-12-10T07:26:26,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586365 2024-12-10T07:26:26,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586365 2024-12-10T07:26:26,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586365) roll requested 2024-12-10T07:26:26,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586372 2024-12-10T07:26:26,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586372 2024-12-10T07:26:26,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586372) roll requested 2024-12-10T07:26:26,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586381 2024-12-10T07:26:26,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586381 2024-12-10T07:26:26,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586381) roll requested 2024-12-10T07:26:26,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586390 2024-12-10T07:26:26,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586390 2024-12-10T07:26:26,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586390) roll requested 2024-12-10T07:26:26,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586400 2024-12-10T07:26:26,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586400 2024-12-10T07:26:26,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586400) roll requested 2024-12-10T07:26:26,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586408 2024-12-10T07:26:26,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586408 2024-12-10T07:26:26,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586408) roll requested 2024-12-10T07:26:26,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586415 2024-12-10T07:26:26,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586415 2024-12-10T07:26:26,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586415) roll requested 2024-12-10T07:26:26,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586423 2024-12-10T07:26:26,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586423 2024-12-10T07:26:26,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586423) roll requested 2024-12-10T07:26:26,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586431 2024-12-10T07:26:26,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586431 2024-12-10T07:26:26,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586431) roll requested 2024-12-10T07:26:26,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586439 2024-12-10T07:26:26,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586439 2024-12-10T07:26:26,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586439) roll requested 2024-12-10T07:26:26,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586449 2024-12-10T07:26:26,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586449 2024-12-10T07:26:26,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586449) roll requested 2024-12-10T07:26:26,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586459 2024-12-10T07:26:26,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586459 2024-12-10T07:26:26,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586459) roll requested 2024-12-10T07:26:26,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586468 2024-12-10T07:26:26,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586468 2024-12-10T07:26:26,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586468) roll requested 2024-12-10T07:26:26,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586476 2024-12-10T07:26:26,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586476 2024-12-10T07:26:26,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586476) roll requested 2024-12-10T07:26:26,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586483 2024-12-10T07:26:26,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586483 2024-12-10T07:26:26,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586483) roll requested 2024-12-10T07:26:26,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586491 2024-12-10T07:26:26,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586491 2024-12-10T07:26:26,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586491) roll requested 2024-12-10T07:26:26,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586499 2024-12-10T07:26:26,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586499 2024-12-10T07:26:26,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586499) roll requested 2024-12-10T07:26:26,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586504 2024-12-10T07:26:26,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586504 2024-12-10T07:26:26,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586504) roll requested 2024-12-10T07:26:26,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586510 2024-12-10T07:26:26,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586510 2024-12-10T07:26:26,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586510) roll requested 2024-12-10T07:26:26,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586520 2024-12-10T07:26:26,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586520 2024-12-10T07:26:26,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586520) roll requested 2024-12-10T07:26:26,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586526 2024-12-10T07:26:26,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586526 2024-12-10T07:26:26,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586526) roll requested 2024-12-10T07:26:26,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586531 2024-12-10T07:26:26,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586531 2024-12-10T07:26:26,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586531) roll requested 2024-12-10T07:26:26,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586537 2024-12-10T07:26:26,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586537 2024-12-10T07:26:26,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586537) roll requested 2024-12-10T07:26:26,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586544 2024-12-10T07:26:26,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586544 2024-12-10T07:26:26,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586544) roll requested 2024-12-10T07:26:26,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586551 2024-12-10T07:26:26,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586551 2024-12-10T07:26:26,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586551) roll requested 2024-12-10T07:26:26,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586559 2024-12-10T07:26:26,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586559 2024-12-10T07:26:26,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586559) roll requested 2024-12-10T07:26:26,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586568 2024-12-10T07:26:26,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586568 2024-12-10T07:26:26,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586568) roll requested 2024-12-10T07:26:26,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586574 2024-12-10T07:26:26,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586574 2024-12-10T07:26:26,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586574) roll requested 2024-12-10T07:26:26,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586581 2024-12-10T07:26:26,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586581 2024-12-10T07:26:26,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586581) roll requested 2024-12-10T07:26:26,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586590 2024-12-10T07:26:26,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586590 2024-12-10T07:26:26,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586590) roll requested 2024-12-10T07:26:26,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586599 2024-12-10T07:26:26,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586599 2024-12-10T07:26:26,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586599) roll requested 2024-12-10T07:26:26,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586607 2024-12-10T07:26:26,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586607 2024-12-10T07:26:26,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586607) roll requested 2024-12-10T07:26:26,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586615 2024-12-10T07:26:26,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586615 2024-12-10T07:26:26,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586615) roll requested 2024-12-10T07:26:26,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586627 2024-12-10T07:26:26,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586627 2024-12-10T07:26:26,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586627) roll requested 2024-12-10T07:26:26,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586634 2024-12-10T07:26:26,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586634 2024-12-10T07:26:26,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586634) roll requested 2024-12-10T07:26:26,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586640 2024-12-10T07:26:26,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586640 2024-12-10T07:26:26,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586640) roll requested 2024-12-10T07:26:26,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586647 2024-12-10T07:26:26,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586647 2024-12-10T07:26:26,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586647) roll requested 2024-12-10T07:26:26,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586655 2024-12-10T07:26:26,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586655 2024-12-10T07:26:26,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586655) roll requested 2024-12-10T07:26:26,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586663 2024-12-10T07:26:26,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586663 2024-12-10T07:26:26,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586663) roll requested 2024-12-10T07:26:26,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586672 2024-12-10T07:26:26,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586672 2024-12-10T07:26:26,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586672) roll requested 2024-12-10T07:26:26,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586679 2024-12-10T07:26:26,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586679 2024-12-10T07:26:26,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586679) roll requested 2024-12-10T07:26:26,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586688 2024-12-10T07:26:26,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586688 2024-12-10T07:26:26,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586688) roll requested 2024-12-10T07:26:26,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586696 2024-12-10T07:26:26,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586696 2024-12-10T07:26:26,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586696) roll requested 2024-12-10T07:26:26,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586705 2024-12-10T07:26:26,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586705 2024-12-10T07:26:26,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586705) roll requested 2024-12-10T07:26:26,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586713 2024-12-10T07:26:26,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586713 2024-12-10T07:26:26,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586713) roll requested 2024-12-10T07:26:26,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586719 2024-12-10T07:26:26,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586719 2024-12-10T07:26:26,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586719) roll requested 2024-12-10T07:26:26,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586727 2024-12-10T07:26:26,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586727 2024-12-10T07:26:26,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586727) roll requested 2024-12-10T07:26:26,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586736 2024-12-10T07:26:26,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586736 2024-12-10T07:26:26,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586736) roll requested 2024-12-10T07:26:26,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586745 2024-12-10T07:26:26,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586745 2024-12-10T07:26:26,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586745) roll requested 2024-12-10T07:26:26,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586752 2024-12-10T07:26:26,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586752 2024-12-10T07:26:26,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586752) roll requested 2024-12-10T07:26:26,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586759 2024-12-10T07:26:26,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586759 2024-12-10T07:26:26,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586759) roll requested 2024-12-10T07:26:26,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586768 2024-12-10T07:26:26,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586768 2024-12-10T07:26:26,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586768) roll requested 2024-12-10T07:26:26,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586776 2024-12-10T07:26:26,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586776 2024-12-10T07:26:26,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586776) roll requested 2024-12-10T07:26:26,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586782 2024-12-10T07:26:26,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586782 2024-12-10T07:26:26,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586782) roll requested 2024-12-10T07:26:26,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586788 2024-12-10T07:26:26,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586788 2024-12-10T07:26:26,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586788) roll requested 2024-12-10T07:26:26,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586796 2024-12-10T07:26:26,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586796 2024-12-10T07:26:26,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586796) roll requested 2024-12-10T07:26:26,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586804 2024-12-10T07:26:26,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586804 2024-12-10T07:26:26,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586804) roll requested 2024-12-10T07:26:26,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586812 2024-12-10T07:26:26,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586812 2024-12-10T07:26:26,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586812) roll requested 2024-12-10T07:26:26,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586820 2024-12-10T07:26:26,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586820 2024-12-10T07:26:26,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586820) roll requested 2024-12-10T07:26:26,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586827 2024-12-10T07:26:26,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586827 2024-12-10T07:26:26,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586827) roll requested 2024-12-10T07:26:26,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586833 2024-12-10T07:26:26,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586833 2024-12-10T07:26:26,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586833) roll requested 2024-12-10T07:26:26,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586840 2024-12-10T07:26:26,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586840 2024-12-10T07:26:26,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586840) roll requested 2024-12-10T07:26:26,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586848 2024-12-10T07:26:26,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586848 2024-12-10T07:26:26,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586848) roll requested 2024-12-10T07:26:26,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586857 2024-12-10T07:26:26,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586857 2024-12-10T07:26:26,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586857) roll requested 2024-12-10T07:26:26,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586864 2024-12-10T07:26:26,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586864 2024-12-10T07:26:26,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586864) roll requested 2024-12-10T07:26:26,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586871 2024-12-10T07:26:26,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586871 2024-12-10T07:26:26,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586871) roll requested 2024-12-10T07:26:26,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586879 2024-12-10T07:26:26,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586879 2024-12-10T07:26:26,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586879) roll requested 2024-12-10T07:26:26,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586886 2024-12-10T07:26:26,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586886 2024-12-10T07:26:26,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586886) roll requested 2024-12-10T07:26:26,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586894 2024-12-10T07:26:26,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586894 2024-12-10T07:26:26,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586894) roll requested 2024-12-10T07:26:26,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586901 2024-12-10T07:26:26,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586901 2024-12-10T07:26:26,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586901) roll requested 2024-12-10T07:26:26,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586910 2024-12-10T07:26:26,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586910 2024-12-10T07:26:26,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586910) roll requested 2024-12-10T07:26:26,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586917 2024-12-10T07:26:26,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586917 2024-12-10T07:26:26,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586917) roll requested 2024-12-10T07:26:26,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586925 2024-12-10T07:26:26,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586925 2024-12-10T07:26:26,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586925) roll requested 2024-12-10T07:26:26,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586933 2024-12-10T07:26:26,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586933 2024-12-10T07:26:26,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586933) roll requested 2024-12-10T07:26:26,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586940 2024-12-10T07:26:26,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586940 2024-12-10T07:26:26,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586940) roll requested 2024-12-10T07:26:26,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586947 2024-12-10T07:26:26,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586947 2024-12-10T07:26:26,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586947) roll requested 2024-12-10T07:26:26,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586953 2024-12-10T07:26:26,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586953 2024-12-10T07:26:26,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586953) roll requested 2024-12-10T07:26:26,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586961 2024-12-10T07:26:26,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586961 2024-12-10T07:26:26,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586961) roll requested 2024-12-10T07:26:26,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586968 2024-12-10T07:26:26,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586968 2024-12-10T07:26:26,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586968) roll requested 2024-12-10T07:26:26,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586973 2024-12-10T07:26:26,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586973 2024-12-10T07:26:26,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586973) roll requested 2024-12-10T07:26:26,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586979 2024-12-10T07:26:26,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586979 2024-12-10T07:26:26,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586979) roll requested 2024-12-10T07:26:26,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586987 2024-12-10T07:26:26,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:26,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586987 2024-12-10T07:26:26,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:26,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:26,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586987) roll requested 2024-12-10T07:26:26,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815586995 2024-12-10T07:26:27,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586995 2024-12-10T07:26:27,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815586995) roll requested 2024-12-10T07:26:27,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587003 2024-12-10T07:26:27,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815586995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587003 2024-12-10T07:26:27,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587003) roll requested 2024-12-10T07:26:27,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587012 2024-12-10T07:26:27,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587012 2024-12-10T07:26:27,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587012) roll requested 2024-12-10T07:26:27,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587022 2024-12-10T07:26:27,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587022 2024-12-10T07:26:27,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587022) roll requested 2024-12-10T07:26:27,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587031 2024-12-10T07:26:27,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587031 2024-12-10T07:26:27,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587031) roll requested 2024-12-10T07:26:27,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587041 2024-12-10T07:26:27,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587041 2024-12-10T07:26:27,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587041) roll requested 2024-12-10T07:26:27,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587046 2024-12-10T07:26:27,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587046 2024-12-10T07:26:27,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587046) roll requested 2024-12-10T07:26:27,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587051 2024-12-10T07:26:27,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587051 2024-12-10T07:26:27,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587051) roll requested 2024-12-10T07:26:27,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587056 2024-12-10T07:26:27,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587056 2024-12-10T07:26:27,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587056) roll requested 2024-12-10T07:26:27,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587062 2024-12-10T07:26:27,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587062 2024-12-10T07:26:27,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587062) roll requested 2024-12-10T07:26:27,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587069 2024-12-10T07:26:27,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587069 2024-12-10T07:26:27,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587069) roll requested 2024-12-10T07:26:27,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587075 2024-12-10T07:26:27,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587075 2024-12-10T07:26:27,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587075) roll requested 2024-12-10T07:26:27,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587082 2024-12-10T07:26:27,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587082 2024-12-10T07:26:27,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587082) roll requested 2024-12-10T07:26:27,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587089 2024-12-10T07:26:27,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587089 2024-12-10T07:26:27,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587089) roll requested 2024-12-10T07:26:27,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587097 2024-12-10T07:26:27,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587097 2024-12-10T07:26:27,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587097) roll requested 2024-12-10T07:26:27,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587106 2024-12-10T07:26:27,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587106 2024-12-10T07:26:27,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587106) roll requested 2024-12-10T07:26:27,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587114 2024-12-10T07:26:27,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587114 2024-12-10T07:26:27,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587114) roll requested 2024-12-10T07:26:27,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587123 2024-12-10T07:26:27,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587123 2024-12-10T07:26:27,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587123) roll requested 2024-12-10T07:26:27,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587132 2024-12-10T07:26:27,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587132 2024-12-10T07:26:27,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587132) roll requested 2024-12-10T07:26:27,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587140 2024-12-10T07:26:27,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587140 2024-12-10T07:26:27,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587140) roll requested 2024-12-10T07:26:27,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587147 2024-12-10T07:26:27,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587147 2024-12-10T07:26:27,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587147) roll requested 2024-12-10T07:26:27,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587156 2024-12-10T07:26:27,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587156 2024-12-10T07:26:27,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587156) roll requested 2024-12-10T07:26:27,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587165 2024-12-10T07:26:27,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587165 2024-12-10T07:26:27,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587165) roll requested 2024-12-10T07:26:27,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587172 2024-12-10T07:26:27,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587172 2024-12-10T07:26:27,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587172) roll requested 2024-12-10T07:26:27,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587182 2024-12-10T07:26:27,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587182 2024-12-10T07:26:27,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587182) roll requested 2024-12-10T07:26:27,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587192 2024-12-10T07:26:27,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587192 2024-12-10T07:26:27,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587192) roll requested 2024-12-10T07:26:27,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587201 2024-12-10T07:26:27,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587201 2024-12-10T07:26:27,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587201) roll requested 2024-12-10T07:26:27,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587206 2024-12-10T07:26:27,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587206 2024-12-10T07:26:27,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587206) roll requested 2024-12-10T07:26:27,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587212 2024-12-10T07:26:27,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587212 2024-12-10T07:26:27,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587212) roll requested 2024-12-10T07:26:27,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587218 2024-12-10T07:26:27,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587218 2024-12-10T07:26:27,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587218) roll requested 2024-12-10T07:26:27,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587223 2024-12-10T07:26:27,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587223 2024-12-10T07:26:27,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587223) roll requested 2024-12-10T07:26:27,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587228 2024-12-10T07:26:27,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587228 2024-12-10T07:26:27,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587228) roll requested 2024-12-10T07:26:27,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587234 2024-12-10T07:26:27,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587234 2024-12-10T07:26:27,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587234) roll requested 2024-12-10T07:26:27,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587242 2024-12-10T07:26:27,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587242 2024-12-10T07:26:27,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587242) roll requested 2024-12-10T07:26:27,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587249 2024-12-10T07:26:27,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587249 2024-12-10T07:26:27,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587249) roll requested 2024-12-10T07:26:27,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587256 2024-12-10T07:26:27,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587256 2024-12-10T07:26:27,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587256) roll requested 2024-12-10T07:26:27,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587263 2024-12-10T07:26:27,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587263 2024-12-10T07:26:27,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587263) roll requested 2024-12-10T07:26:27,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587271 2024-12-10T07:26:27,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587271 2024-12-10T07:26:27,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587271) roll requested 2024-12-10T07:26:27,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587280 2024-12-10T07:26:27,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587280 2024-12-10T07:26:27,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587280) roll requested 2024-12-10T07:26:27,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587288 2024-12-10T07:26:27,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587288 2024-12-10T07:26:27,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587288) roll requested 2024-12-10T07:26:27,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587296 2024-12-10T07:26:27,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587296 2024-12-10T07:26:27,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587296) roll requested 2024-12-10T07:26:27,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587304 2024-12-10T07:26:27,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587304 2024-12-10T07:26:27,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587304) roll requested 2024-12-10T07:26:27,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587311 2024-12-10T07:26:27,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587311 2024-12-10T07:26:27,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587311) roll requested 2024-12-10T07:26:27,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587319 2024-12-10T07:26:27,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587319 2024-12-10T07:26:27,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587319) roll requested 2024-12-10T07:26:27,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587328 2024-12-10T07:26:27,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587328 2024-12-10T07:26:27,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587328) roll requested 2024-12-10T07:26:27,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587336 2024-12-10T07:26:27,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587336 2024-12-10T07:26:27,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587336) roll requested 2024-12-10T07:26:27,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587344 2024-12-10T07:26:27,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587344 2024-12-10T07:26:27,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587344) roll requested 2024-12-10T07:26:27,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587352 2024-12-10T07:26:27,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587352 2024-12-10T07:26:27,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587352) roll requested 2024-12-10T07:26:27,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587360 2024-12-10T07:26:27,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587360 2024-12-10T07:26:27,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587360) roll requested 2024-12-10T07:26:27,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587369 2024-12-10T07:26:27,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587369 2024-12-10T07:26:27,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587369) roll requested 2024-12-10T07:26:27,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587375 2024-12-10T07:26:27,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587375 2024-12-10T07:26:27,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587375) roll requested 2024-12-10T07:26:27,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587382 2024-12-10T07:26:27,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587382 2024-12-10T07:26:27,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587382) roll requested 2024-12-10T07:26:27,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587391 2024-12-10T07:26:27,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587391 2024-12-10T07:26:27,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587391) roll requested 2024-12-10T07:26:27,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587401 2024-12-10T07:26:27,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587401 2024-12-10T07:26:27,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587401) roll requested 2024-12-10T07:26:27,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587411 2024-12-10T07:26:27,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587411 2024-12-10T07:26:27,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587411) roll requested 2024-12-10T07:26:27,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587419 2024-12-10T07:26:27,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587419 2024-12-10T07:26:27,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587419) roll requested 2024-12-10T07:26:27,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587427 2024-12-10T07:26:27,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587427 2024-12-10T07:26:27,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587427) roll requested 2024-12-10T07:26:27,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587436 2024-12-10T07:26:27,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587436 2024-12-10T07:26:27,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587436) roll requested 2024-12-10T07:26:27,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587446 2024-12-10T07:26:27,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587446 2024-12-10T07:26:27,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587446) roll requested 2024-12-10T07:26:27,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587451 2024-12-10T07:26:27,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587451 2024-12-10T07:26:27,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587451) roll requested 2024-12-10T07:26:27,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587457 2024-12-10T07:26:27,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587457 2024-12-10T07:26:27,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587457) roll requested 2024-12-10T07:26:27,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587465 2024-12-10T07:26:27,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587465 2024-12-10T07:26:27,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587465) roll requested 2024-12-10T07:26:27,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587472 2024-12-10T07:26:27,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587472 2024-12-10T07:26:27,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587472) roll requested 2024-12-10T07:26:27,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587480 2024-12-10T07:26:27,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587480 2024-12-10T07:26:27,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587480) roll requested 2024-12-10T07:26:27,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587488 2024-12-10T07:26:27,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587488 2024-12-10T07:26:27,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587488) roll requested 2024-12-10T07:26:27,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587497 2024-12-10T07:26:27,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587497 2024-12-10T07:26:27,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587497) roll requested 2024-12-10T07:26:27,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587506 2024-12-10T07:26:27,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587506 2024-12-10T07:26:27,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587506) roll requested 2024-12-10T07:26:27,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587515 2024-12-10T07:26:27,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587515 2024-12-10T07:26:27,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587515) roll requested 2024-12-10T07:26:27,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587523 2024-12-10T07:26:27,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587523 2024-12-10T07:26:27,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587523) roll requested 2024-12-10T07:26:27,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587531 2024-12-10T07:26:27,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587531 2024-12-10T07:26:27,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587531) roll requested 2024-12-10T07:26:27,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587540 2024-12-10T07:26:27,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587540 2024-12-10T07:26:27,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587540) roll requested 2024-12-10T07:26:27,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587550 2024-12-10T07:26:27,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587550 2024-12-10T07:26:27,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587550) roll requested 2024-12-10T07:26:27,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587556 2024-12-10T07:26:27,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587556 2024-12-10T07:26:27,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587556) roll requested 2024-12-10T07:26:27,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587561 2024-12-10T07:26:27,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587561 2024-12-10T07:26:27,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587561) roll requested 2024-12-10T07:26:27,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587567 2024-12-10T07:26:27,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587567 2024-12-10T07:26:27,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587567) roll requested 2024-12-10T07:26:27,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587572 2024-12-10T07:26:27,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587572 2024-12-10T07:26:27,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587572) roll requested 2024-12-10T07:26:27,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587579 2024-12-10T07:26:27,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587579 2024-12-10T07:26:27,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587579) roll requested 2024-12-10T07:26:27,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587587 2024-12-10T07:26:27,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587587 2024-12-10T07:26:27,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587587) roll requested 2024-12-10T07:26:27,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587595 2024-12-10T07:26:27,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587595 2024-12-10T07:26:27,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587595) roll requested 2024-12-10T07:26:27,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587604 2024-12-10T07:26:27,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587604 2024-12-10T07:26:27,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587604) roll requested 2024-12-10T07:26:27,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587612 2024-12-10T07:26:27,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587612 2024-12-10T07:26:27,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587612) roll requested 2024-12-10T07:26:27,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587620 2024-12-10T07:26:27,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587620 2024-12-10T07:26:27,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587620) roll requested 2024-12-10T07:26:27,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587628 2024-12-10T07:26:27,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587628 2024-12-10T07:26:27,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587628) roll requested 2024-12-10T07:26:27,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587637 2024-12-10T07:26:27,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587637 2024-12-10T07:26:27,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587637) roll requested 2024-12-10T07:26:27,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587647 2024-12-10T07:26:27,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587647 2024-12-10T07:26:27,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587647) roll requested 2024-12-10T07:26:27,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587658 2024-12-10T07:26:27,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587658 2024-12-10T07:26:27,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587658) roll requested 2024-12-10T07:26:27,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587667 2024-12-10T07:26:27,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587667 2024-12-10T07:26:27,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587667) roll requested 2024-12-10T07:26:27,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587673 2024-12-10T07:26:27,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587673 2024-12-10T07:26:27,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587673) roll requested 2024-12-10T07:26:27,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587678 2024-12-10T07:26:27,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587678 2024-12-10T07:26:27,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587678) roll requested 2024-12-10T07:26:27,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587685 2024-12-10T07:26:27,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587685 2024-12-10T07:26:27,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587685) roll requested 2024-12-10T07:26:27,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587692 2024-12-10T07:26:27,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587692 2024-12-10T07:26:27,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587692) roll requested 2024-12-10T07:26:27,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587699 2024-12-10T07:26:27,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587699 2024-12-10T07:26:27,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587699) roll requested 2024-12-10T07:26:27,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587708 2024-12-10T07:26:27,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587708 2024-12-10T07:26:27,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587708) roll requested 2024-12-10T07:26:27,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587717 2024-12-10T07:26:27,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587717 2024-12-10T07:26:27,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587717) roll requested 2024-12-10T07:26:27,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587726 2024-12-10T07:26:27,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587726 2024-12-10T07:26:27,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587726) roll requested 2024-12-10T07:26:27,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587731 2024-12-10T07:26:27,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587731 2024-12-10T07:26:27,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587731) roll requested 2024-12-10T07:26:27,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587740 2024-12-10T07:26:27,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587740 2024-12-10T07:26:27,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587740) roll requested 2024-12-10T07:26:27,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587748 2024-12-10T07:26:27,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587748 2024-12-10T07:26:27,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587748) roll requested 2024-12-10T07:26:27,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587756 2024-12-10T07:26:27,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587756 2024-12-10T07:26:27,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587756) roll requested 2024-12-10T07:26:27,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587765 2024-12-10T07:26:27,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587765 2024-12-10T07:26:27,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587765) roll requested 2024-12-10T07:26:27,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587775 2024-12-10T07:26:27,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587775 2024-12-10T07:26:27,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587775) roll requested 2024-12-10T07:26:27,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587783 2024-12-10T07:26:27,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587783 2024-12-10T07:26:27,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587783) roll requested 2024-12-10T07:26:27,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587792 2024-12-10T07:26:27,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587792 2024-12-10T07:26:27,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587792) roll requested 2024-12-10T07:26:27,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587801 2024-12-10T07:26:27,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587801 2024-12-10T07:26:27,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587801) roll requested 2024-12-10T07:26:27,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587811 2024-12-10T07:26:27,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587811 2024-12-10T07:26:27,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587811) roll requested 2024-12-10T07:26:27,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587818 2024-12-10T07:26:27,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587818 2024-12-10T07:26:27,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587818) roll requested 2024-12-10T07:26:27,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587825 2024-12-10T07:26:27,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587825 2024-12-10T07:26:27,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587825) roll requested 2024-12-10T07:26:27,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587832 2024-12-10T07:26:27,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587832 2024-12-10T07:26:27,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587832) roll requested 2024-12-10T07:26:27,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587839 2024-12-10T07:26:27,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587839 2024-12-10T07:26:27,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587839) roll requested 2024-12-10T07:26:27,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587848 2024-12-10T07:26:27,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587848 2024-12-10T07:26:27,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587848) roll requested 2024-12-10T07:26:27,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587857 2024-12-10T07:26:27,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587857 2024-12-10T07:26:27,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587857) roll requested 2024-12-10T07:26:27,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587866 2024-12-10T07:26:27,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587866 2024-12-10T07:26:27,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587866) roll requested 2024-12-10T07:26:27,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587873 2024-12-10T07:26:27,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587873 2024-12-10T07:26:27,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587873) roll requested 2024-12-10T07:26:27,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587882 2024-12-10T07:26:27,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587882 2024-12-10T07:26:27,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587882) roll requested 2024-12-10T07:26:27,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587891 2024-12-10T07:26:27,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587891 2024-12-10T07:26:27,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587891) roll requested 2024-12-10T07:26:27,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587900 2024-12-10T07:26:27,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587900 2024-12-10T07:26:27,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587900) roll requested 2024-12-10T07:26:27,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587907 2024-12-10T07:26:27,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587907 2024-12-10T07:26:27,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587907) roll requested 2024-12-10T07:26:27,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587915 2024-12-10T07:26:27,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587915 2024-12-10T07:26:27,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587915) roll requested 2024-12-10T07:26:27,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587923 2024-12-10T07:26:27,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587923 2024-12-10T07:26:27,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587923) roll requested 2024-12-10T07:26:27,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587932 2024-12-10T07:26:27,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587932 2024-12-10T07:26:27,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587932) roll requested 2024-12-10T07:26:27,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587941 2024-12-10T07:26:27,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587941 2024-12-10T07:26:27,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587941) roll requested 2024-12-10T07:26:27,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587949 2024-12-10T07:26:27,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587949 2024-12-10T07:26:27,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587949) roll requested 2024-12-10T07:26:27,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587955 2024-12-10T07:26:27,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587955 2024-12-10T07:26:27,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587955) roll requested 2024-12-10T07:26:27,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587963 2024-12-10T07:26:27,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587963 2024-12-10T07:26:27,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587963) roll requested 2024-12-10T07:26:27,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587971 2024-12-10T07:26:27,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587971 2024-12-10T07:26:27,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587971) roll requested 2024-12-10T07:26:27,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587978 2024-12-10T07:26:27,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587978 2024-12-10T07:26:27,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587978) roll requested 2024-12-10T07:26:27,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587987 2024-12-10T07:26:27,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:27,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587987 2024-12-10T07:26:27,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:27,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:27,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587987) roll requested 2024-12-10T07:26:27,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815587994 2024-12-10T07:26:28,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587994 2024-12-10T07:26:28,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815587994) roll requested 2024-12-10T07:26:28,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588004 2024-12-10T07:26:28,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815587994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588004 2024-12-10T07:26:28,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588004) roll requested 2024-12-10T07:26:28,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588012 2024-12-10T07:26:28,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588012 2024-12-10T07:26:28,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588012) roll requested 2024-12-10T07:26:28,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588021 2024-12-10T07:26:28,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588021 2024-12-10T07:26:28,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588021) roll requested 2024-12-10T07:26:28,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588030 2024-12-10T07:26:28,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588030 2024-12-10T07:26:28,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588030) roll requested 2024-12-10T07:26:28,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588041 2024-12-10T07:26:28,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588041 2024-12-10T07:26:28,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588041) roll requested 2024-12-10T07:26:28,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588049 2024-12-10T07:26:28,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588049 2024-12-10T07:26:28,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588049) roll requested 2024-12-10T07:26:28,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588054 2024-12-10T07:26:28,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588054 2024-12-10T07:26:28,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588054) roll requested 2024-12-10T07:26:28,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588059 2024-12-10T07:26:28,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588059 2024-12-10T07:26:28,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588059) roll requested 2024-12-10T07:26:28,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588065 2024-12-10T07:26:28,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588065 2024-12-10T07:26:28,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588065) roll requested 2024-12-10T07:26:28,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588072 2024-12-10T07:26:28,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588072 2024-12-10T07:26:28,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588072) roll requested 2024-12-10T07:26:28,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588080 2024-12-10T07:26:28,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588080 2024-12-10T07:26:28,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588080) roll requested 2024-12-10T07:26:28,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588088 2024-12-10T07:26:28,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588088 2024-12-10T07:26:28,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588088) roll requested 2024-12-10T07:26:28,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588096 2024-12-10T07:26:28,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588096 2024-12-10T07:26:28,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588096) roll requested 2024-12-10T07:26:28,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588104 2024-12-10T07:26:28,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588104 2024-12-10T07:26:28,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588104) roll requested 2024-12-10T07:26:28,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588111 2024-12-10T07:26:28,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588111 2024-12-10T07:26:28,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588111) roll requested 2024-12-10T07:26:28,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588117 2024-12-10T07:26:28,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588117 2024-12-10T07:26:28,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588117) roll requested 2024-12-10T07:26:28,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588124 2024-12-10T07:26:28,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588124 2024-12-10T07:26:28,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588124) roll requested 2024-12-10T07:26:28,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588131 2024-12-10T07:26:28,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588131 2024-12-10T07:26:28,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588131) roll requested 2024-12-10T07:26:28,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588141 2024-12-10T07:26:28,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588141 2024-12-10T07:26:28,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588141) roll requested 2024-12-10T07:26:28,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588151 2024-12-10T07:26:28,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588151 2024-12-10T07:26:28,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588151) roll requested 2024-12-10T07:26:28,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588159 2024-12-10T07:26:28,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588159 2024-12-10T07:26:28,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588159) roll requested 2024-12-10T07:26:28,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588165 2024-12-10T07:26:28,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588165 2024-12-10T07:26:28,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588165) roll requested 2024-12-10T07:26:28,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588170 2024-12-10T07:26:28,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588170 2024-12-10T07:26:28,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588170) roll requested 2024-12-10T07:26:28,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588175 2024-12-10T07:26:28,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588175 2024-12-10T07:26:28,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588175) roll requested 2024-12-10T07:26:28,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588180 2024-12-10T07:26:28,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588180 2024-12-10T07:26:28,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588180) roll requested 2024-12-10T07:26:28,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588186 2024-12-10T07:26:28,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588186 2024-12-10T07:26:28,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588186) roll requested 2024-12-10T07:26:28,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588191 2024-12-10T07:26:28,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588191 2024-12-10T07:26:28,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588191) roll requested 2024-12-10T07:26:28,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588196 2024-12-10T07:26:28,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588196 2024-12-10T07:26:28,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588196) roll requested 2024-12-10T07:26:28,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588202 2024-12-10T07:26:28,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588202 2024-12-10T07:26:28,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588202) roll requested 2024-12-10T07:26:28,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588209 2024-12-10T07:26:28,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588209 2024-12-10T07:26:28,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588209) roll requested 2024-12-10T07:26:28,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588217 2024-12-10T07:26:28,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588217 2024-12-10T07:26:28,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588217) roll requested 2024-12-10T07:26:28,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588224 2024-12-10T07:26:28,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588224 2024-12-10T07:26:28,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588224) roll requested 2024-12-10T07:26:28,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588233 2024-12-10T07:26:28,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588233 2024-12-10T07:26:28,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588233) roll requested 2024-12-10T07:26:28,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588243 2024-12-10T07:26:28,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588243 2024-12-10T07:26:28,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588243) roll requested 2024-12-10T07:26:28,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588252 2024-12-10T07:26:28,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588252 2024-12-10T07:26:28,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588252) roll requested 2024-12-10T07:26:28,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588259 2024-12-10T07:26:28,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588259 2024-12-10T07:26:28,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588259) roll requested 2024-12-10T07:26:28,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588267 2024-12-10T07:26:28,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588267 2024-12-10T07:26:28,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588267) roll requested 2024-12-10T07:26:28,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588275 2024-12-10T07:26:28,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588275 2024-12-10T07:26:28,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588275) roll requested 2024-12-10T07:26:28,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588284 2024-12-10T07:26:28,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588284 2024-12-10T07:26:28,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588284) roll requested 2024-12-10T07:26:28,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588293 2024-12-10T07:26:28,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588293 2024-12-10T07:26:28,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588293) roll requested 2024-12-10T07:26:28,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588302 2024-12-10T07:26:28,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588302 2024-12-10T07:26:28,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588302) roll requested 2024-12-10T07:26:28,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588310 2024-12-10T07:26:28,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588310 2024-12-10T07:26:28,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588310) roll requested 2024-12-10T07:26:28,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588319 2024-12-10T07:26:28,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588319 2024-12-10T07:26:28,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588319) roll requested 2024-12-10T07:26:28,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588327 2024-12-10T07:26:28,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588327 2024-12-10T07:26:28,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588327) roll requested 2024-12-10T07:26:28,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588334 2024-12-10T07:26:28,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588334 2024-12-10T07:26:28,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588334) roll requested 2024-12-10T07:26:28,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588343 2024-12-10T07:26:28,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588343 2024-12-10T07:26:28,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588343) roll requested 2024-12-10T07:26:28,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588354 2024-12-10T07:26:28,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588354 2024-12-10T07:26:28,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588354) roll requested 2024-12-10T07:26:28,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588359 2024-12-10T07:26:28,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588359 2024-12-10T07:26:28,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588359) roll requested 2024-12-10T07:26:28,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588363 2024-12-10T07:26:28,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588363 2024-12-10T07:26:28,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588363) roll requested 2024-12-10T07:26:28,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588368 2024-12-10T07:26:28,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588368 2024-12-10T07:26:28,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588368) roll requested 2024-12-10T07:26:28,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588374 2024-12-10T07:26:28,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588374 2024-12-10T07:26:28,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588374) roll requested 2024-12-10T07:26:28,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588380 2024-12-10T07:26:28,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588380 2024-12-10T07:26:28,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588380) roll requested 2024-12-10T07:26:28,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588387 2024-12-10T07:26:28,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588387 2024-12-10T07:26:28,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588387) roll requested 2024-12-10T07:26:28,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588396 2024-12-10T07:26:28,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588396 2024-12-10T07:26:28,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588396) roll requested 2024-12-10T07:26:28,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588404 2024-12-10T07:26:28,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588404 2024-12-10T07:26:28,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588404) roll requested 2024-12-10T07:26:28,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588413 2024-12-10T07:26:28,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588413 2024-12-10T07:26:28,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588413) roll requested 2024-12-10T07:26:28,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588422 2024-12-10T07:26:28,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588422 2024-12-10T07:26:28,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588422) roll requested 2024-12-10T07:26:28,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588431 2024-12-10T07:26:28,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588431 2024-12-10T07:26:28,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588431) roll requested 2024-12-10T07:26:28,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588439 2024-12-10T07:26:28,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588439 2024-12-10T07:26:28,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588439) roll requested 2024-12-10T07:26:28,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588448 2024-12-10T07:26:28,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588448 2024-12-10T07:26:28,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588448) roll requested 2024-12-10T07:26:28,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588457 2024-12-10T07:26:28,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588457 2024-12-10T07:26:28,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588457) roll requested 2024-12-10T07:26:28,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588465 2024-12-10T07:26:28,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588465 2024-12-10T07:26:28,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588465) roll requested 2024-12-10T07:26:28,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588472 2024-12-10T07:26:28,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588472 2024-12-10T07:26:28,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588472) roll requested 2024-12-10T07:26:28,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588479 2024-12-10T07:26:28,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588479 2024-12-10T07:26:28,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588479) roll requested 2024-12-10T07:26:28,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588487 2024-12-10T07:26:28,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588487 2024-12-10T07:26:28,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588487) roll requested 2024-12-10T07:26:28,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588493 2024-12-10T07:26:28,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588493 2024-12-10T07:26:28,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588493) roll requested 2024-12-10T07:26:28,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588500 2024-12-10T07:26:28,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588500 2024-12-10T07:26:28,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588500) roll requested 2024-12-10T07:26:28,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588507 2024-12-10T07:26:28,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588507 2024-12-10T07:26:28,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588507) roll requested 2024-12-10T07:26:28,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588515 2024-12-10T07:26:28,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588515 2024-12-10T07:26:28,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588515) roll requested 2024-12-10T07:26:28,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588522 2024-12-10T07:26:28,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588522 2024-12-10T07:26:28,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588522) roll requested 2024-12-10T07:26:28,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588530 2024-12-10T07:26:28,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588530 2024-12-10T07:26:28,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588530) roll requested 2024-12-10T07:26:28,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588538 2024-12-10T07:26:28,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588538 2024-12-10T07:26:28,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588538) roll requested 2024-12-10T07:26:28,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588547 2024-12-10T07:26:28,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588547 2024-12-10T07:26:28,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588547) roll requested 2024-12-10T07:26:28,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588557 2024-12-10T07:26:28,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588557 2024-12-10T07:26:28,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588557) roll requested 2024-12-10T07:26:28,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588564 2024-12-10T07:26:28,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588564 2024-12-10T07:26:28,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588564) roll requested 2024-12-10T07:26:28,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588572 2024-12-10T07:26:28,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588572 2024-12-10T07:26:28,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588572) roll requested 2024-12-10T07:26:28,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588581 2024-12-10T07:26:28,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588581 2024-12-10T07:26:28,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588581) roll requested 2024-12-10T07:26:28,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588589 2024-12-10T07:26:28,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588589 2024-12-10T07:26:28,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588589) roll requested 2024-12-10T07:26:28,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588596 2024-12-10T07:26:28,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588596 2024-12-10T07:26:28,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588596) roll requested 2024-12-10T07:26:28,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588604 2024-12-10T07:26:28,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588604 2024-12-10T07:26:28,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588604) roll requested 2024-12-10T07:26:28,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588611 2024-12-10T07:26:28,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588611 2024-12-10T07:26:28,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588611) roll requested 2024-12-10T07:26:28,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588619 2024-12-10T07:26:28,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588619 2024-12-10T07:26:28,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588619) roll requested 2024-12-10T07:26:28,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588628 2024-12-10T07:26:28,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588628 2024-12-10T07:26:28,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588628) roll requested 2024-12-10T07:26:28,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588637 2024-12-10T07:26:28,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588637 2024-12-10T07:26:28,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588637) roll requested 2024-12-10T07:26:28,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588643 2024-12-10T07:26:28,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588643 2024-12-10T07:26:28,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588643) roll requested 2024-12-10T07:26:28,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588649 2024-12-10T07:26:28,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588649 2024-12-10T07:26:28,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588649) roll requested 2024-12-10T07:26:28,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588657 2024-12-10T07:26:28,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588657 2024-12-10T07:26:28,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588657) roll requested 2024-12-10T07:26:28,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588665 2024-12-10T07:26:28,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588665 2024-12-10T07:26:28,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588665) roll requested 2024-12-10T07:26:28,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588675 2024-12-10T07:26:28,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588675 2024-12-10T07:26:28,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588675) roll requested 2024-12-10T07:26:28,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588680 2024-12-10T07:26:28,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588680 2024-12-10T07:26:28,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588680) roll requested 2024-12-10T07:26:28,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588685 2024-12-10T07:26:28,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588685 2024-12-10T07:26:28,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588685) roll requested 2024-12-10T07:26:28,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588690 2024-12-10T07:26:28,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588690 2024-12-10T07:26:28,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588690) roll requested 2024-12-10T07:26:28,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588695 2024-12-10T07:26:28,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588695 2024-12-10T07:26:28,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588695) roll requested 2024-12-10T07:26:28,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588701 2024-12-10T07:26:28,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588701 2024-12-10T07:26:28,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588701) roll requested 2024-12-10T07:26:28,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588709 2024-12-10T07:26:28,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588709 2024-12-10T07:26:28,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588709) roll requested 2024-12-10T07:26:28,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588716 2024-12-10T07:26:28,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588716 2024-12-10T07:26:28,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588716) roll requested 2024-12-10T07:26:28,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588724 2024-12-10T07:26:28,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588724 2024-12-10T07:26:28,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588724) roll requested 2024-12-10T07:26:28,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588733 2024-12-10T07:26:28,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588733 2024-12-10T07:26:28,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588733) roll requested 2024-12-10T07:26:28,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588741 2024-12-10T07:26:28,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588741 2024-12-10T07:26:28,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588741) roll requested 2024-12-10T07:26:28,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588749 2024-12-10T07:26:28,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588749 2024-12-10T07:26:28,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588749) roll requested 2024-12-10T07:26:28,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588759 2024-12-10T07:26:28,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588759 2024-12-10T07:26:28,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588759) roll requested 2024-12-10T07:26:28,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588768 2024-12-10T07:26:28,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588768 2024-12-10T07:26:28,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588768) roll requested 2024-12-10T07:26:28,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588776 2024-12-10T07:26:28,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588776 2024-12-10T07:26:28,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588776) roll requested 2024-12-10T07:26:28,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588784 2024-12-10T07:26:28,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588784 2024-12-10T07:26:28,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588784) roll requested 2024-12-10T07:26:28,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588791 2024-12-10T07:26:28,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588791 2024-12-10T07:26:28,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588791) roll requested 2024-12-10T07:26:28,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588797 2024-12-10T07:26:28,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588797 2024-12-10T07:26:28,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588797) roll requested 2024-12-10T07:26:28,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588804 2024-12-10T07:26:28,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588804 2024-12-10T07:26:28,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588804) roll requested 2024-12-10T07:26:28,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588812 2024-12-10T07:26:28,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588812 2024-12-10T07:26:28,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588812) roll requested 2024-12-10T07:26:28,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588818 2024-12-10T07:26:28,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588818 2024-12-10T07:26:28,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588818) roll requested 2024-12-10T07:26:28,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588826 2024-12-10T07:26:28,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588826 2024-12-10T07:26:28,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588826) roll requested 2024-12-10T07:26:28,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588834 2024-12-10T07:26:28,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588834 2024-12-10T07:26:28,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588834) roll requested 2024-12-10T07:26:28,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588842 2024-12-10T07:26:28,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588842 2024-12-10T07:26:28,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588842) roll requested 2024-12-10T07:26:28,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588851 2024-12-10T07:26:28,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588851 2024-12-10T07:26:28,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588851) roll requested 2024-12-10T07:26:28,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588860 2024-12-10T07:26:28,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588860 2024-12-10T07:26:28,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588860) roll requested 2024-12-10T07:26:28,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588868 2024-12-10T07:26:28,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588868 2024-12-10T07:26:28,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588868) roll requested 2024-12-10T07:26:28,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588876 2024-12-10T07:26:28,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588876 2024-12-10T07:26:28,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588876) roll requested 2024-12-10T07:26:28,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588884 2024-12-10T07:26:28,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588884 2024-12-10T07:26:28,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588884) roll requested 2024-12-10T07:26:28,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588892 2024-12-10T07:26:28,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588892 2024-12-10T07:26:28,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588892) roll requested 2024-12-10T07:26:28,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588901 2024-12-10T07:26:28,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588901 2024-12-10T07:26:28,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588901) roll requested 2024-12-10T07:26:28,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588910 2024-12-10T07:26:28,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588910 2024-12-10T07:26:28,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588910) roll requested 2024-12-10T07:26:28,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588917 2024-12-10T07:26:28,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588917 2024-12-10T07:26:28,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588917) roll requested 2024-12-10T07:26:28,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588925 2024-12-10T07:26:28,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588925 2024-12-10T07:26:28,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588925) roll requested 2024-12-10T07:26:28,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588933 2024-12-10T07:26:28,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588933 2024-12-10T07:26:28,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588933) roll requested 2024-12-10T07:26:28,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588939 2024-12-10T07:26:28,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588939 2024-12-10T07:26:28,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588939) roll requested 2024-12-10T07:26:28,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588947 2024-12-10T07:26:28,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588947 2024-12-10T07:26:28,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588947) roll requested 2024-12-10T07:26:28,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588955 2024-12-10T07:26:28,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588955 2024-12-10T07:26:28,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588955) roll requested 2024-12-10T07:26:28,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588963 2024-12-10T07:26:28,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588963 2024-12-10T07:26:28,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588963) roll requested 2024-12-10T07:26:28,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588972 2024-12-10T07:26:28,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588972 2024-12-10T07:26:28,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588972) roll requested 2024-12-10T07:26:28,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588979 2024-12-10T07:26:28,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588979 2024-12-10T07:26:28,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588979) roll requested 2024-12-10T07:26:28,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588986 2024-12-10T07:26:28,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:28,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588986 2024-12-10T07:26:28,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:28,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:28,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588986) roll requested 2024-12-10T07:26:28,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815588994 2024-12-10T07:26:28,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588994 2024-12-10T07:26:29,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815588994) roll requested 2024-12-10T07:26:29,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589001 2024-12-10T07:26:29,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815588994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589001 2024-12-10T07:26:29,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589001) roll requested 2024-12-10T07:26:29,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589010 2024-12-10T07:26:29,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589010 2024-12-10T07:26:29,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589010) roll requested 2024-12-10T07:26:29,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589019 2024-12-10T07:26:29,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589019 2024-12-10T07:26:29,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589019) roll requested 2024-12-10T07:26:29,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589027 2024-12-10T07:26:29,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589027 2024-12-10T07:26:29,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589027) roll requested 2024-12-10T07:26:29,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589035 2024-12-10T07:26:29,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589035 2024-12-10T07:26:29,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589035) roll requested 2024-12-10T07:26:29,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589044 2024-12-10T07:26:29,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589044 2024-12-10T07:26:29,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589044) roll requested 2024-12-10T07:26:29,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589052 2024-12-10T07:26:29,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589052 2024-12-10T07:26:29,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589052) roll requested 2024-12-10T07:26:29,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589061 2024-12-10T07:26:29,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589061 2024-12-10T07:26:29,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589061) roll requested 2024-12-10T07:26:29,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589067 2024-12-10T07:26:29,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589067 2024-12-10T07:26:29,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589067) roll requested 2024-12-10T07:26:29,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589073 2024-12-10T07:26:29,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589073 2024-12-10T07:26:29,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589073) roll requested 2024-12-10T07:26:29,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589079 2024-12-10T07:26:29,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589079 2024-12-10T07:26:29,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589079) roll requested 2024-12-10T07:26:29,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589085 2024-12-10T07:26:29,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589085 2024-12-10T07:26:29,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589085) roll requested 2024-12-10T07:26:29,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589093 2024-12-10T07:26:29,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589093 2024-12-10T07:26:29,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589093) roll requested 2024-12-10T07:26:29,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589101 2024-12-10T07:26:29,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589101 2024-12-10T07:26:29,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589101) roll requested 2024-12-10T07:26:29,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589108 2024-12-10T07:26:29,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589108 2024-12-10T07:26:29,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589108) roll requested 2024-12-10T07:26:29,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589114 2024-12-10T07:26:29,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589114 2024-12-10T07:26:29,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589114) roll requested 2024-12-10T07:26:29,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589121 2024-12-10T07:26:29,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589121 2024-12-10T07:26:29,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589121) roll requested 2024-12-10T07:26:29,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589133 2024-12-10T07:26:29,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589133 2024-12-10T07:26:29,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589133) roll requested 2024-12-10T07:26:29,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589139 2024-12-10T07:26:29,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589139 2024-12-10T07:26:29,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589139) roll requested 2024-12-10T07:26:29,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589144 2024-12-10T07:26:29,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589144 2024-12-10T07:26:29,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589144) roll requested 2024-12-10T07:26:29,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589155 2024-12-10T07:26:29,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589155 2024-12-10T07:26:29,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589155) roll requested 2024-12-10T07:26:29,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589161 2024-12-10T07:26:29,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589161 2024-12-10T07:26:29,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589161) roll requested 2024-12-10T07:26:29,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589169 2024-12-10T07:26:29,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589169 2024-12-10T07:26:29,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589169) roll requested 2024-12-10T07:26:29,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589175 2024-12-10T07:26:29,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589175 2024-12-10T07:26:29,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589175) roll requested 2024-12-10T07:26:29,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589181 2024-12-10T07:26:29,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589181 2024-12-10T07:26:29,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589181) roll requested 2024-12-10T07:26:29,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589186 2024-12-10T07:26:29,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589186 2024-12-10T07:26:29,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589186) roll requested 2024-12-10T07:26:29,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589192 2024-12-10T07:26:29,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589192 2024-12-10T07:26:29,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589192) roll requested 2024-12-10T07:26:29,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589197 2024-12-10T07:26:29,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589197 2024-12-10T07:26:29,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589197) roll requested 2024-12-10T07:26:29,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589202 2024-12-10T07:26:29,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589202 2024-12-10T07:26:29,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589202) roll requested 2024-12-10T07:26:29,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589209 2024-12-10T07:26:29,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589209 2024-12-10T07:26:29,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589209) roll requested 2024-12-10T07:26:29,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589218 2024-12-10T07:26:29,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589218 2024-12-10T07:26:29,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589218) roll requested 2024-12-10T07:26:29,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589225 2024-12-10T07:26:29,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589225 2024-12-10T07:26:29,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589225) roll requested 2024-12-10T07:26:29,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589232 2024-12-10T07:26:29,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589232 2024-12-10T07:26:29,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589232) roll requested 2024-12-10T07:26:29,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589239 2024-12-10T07:26:29,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589239 2024-12-10T07:26:29,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589239) roll requested 2024-12-10T07:26:29,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589246 2024-12-10T07:26:29,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589246 2024-12-10T07:26:29,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589246) roll requested 2024-12-10T07:26:29,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589254 2024-12-10T07:26:29,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589254 2024-12-10T07:26:29,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589254) roll requested 2024-12-10T07:26:29,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589262 2024-12-10T07:26:29,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589262 2024-12-10T07:26:29,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589262) roll requested 2024-12-10T07:26:29,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589272 2024-12-10T07:26:29,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589272 2024-12-10T07:26:29,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589272) roll requested 2024-12-10T07:26:29,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589280 2024-12-10T07:26:29,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589280 2024-12-10T07:26:29,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589280) roll requested 2024-12-10T07:26:29,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589288 2024-12-10T07:26:29,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589288 2024-12-10T07:26:29,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589288) roll requested 2024-12-10T07:26:29,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589296 2024-12-10T07:26:29,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589296 2024-12-10T07:26:29,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589296) roll requested 2024-12-10T07:26:29,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589304 2024-12-10T07:26:29,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589304 2024-12-10T07:26:29,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589304) roll requested 2024-12-10T07:26:29,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589313 2024-12-10T07:26:29,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589313 2024-12-10T07:26:29,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589313) roll requested 2024-12-10T07:26:29,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589323 2024-12-10T07:26:29,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589323 2024-12-10T07:26:29,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589323) roll requested 2024-12-10T07:26:29,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589331 2024-12-10T07:26:29,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589331 2024-12-10T07:26:29,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589331) roll requested 2024-12-10T07:26:29,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589338 2024-12-10T07:26:29,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589338 2024-12-10T07:26:29,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589338) roll requested 2024-12-10T07:26:29,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589343 2024-12-10T07:26:29,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589343 2024-12-10T07:26:29,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589343) roll requested 2024-12-10T07:26:29,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589348 2024-12-10T07:26:29,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589348 2024-12-10T07:26:29,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589348) roll requested 2024-12-10T07:26:29,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589357 2024-12-10T07:26:29,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589357 2024-12-10T07:26:29,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589357) roll requested 2024-12-10T07:26:29,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589362 2024-12-10T07:26:29,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589362 2024-12-10T07:26:29,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589362) roll requested 2024-12-10T07:26:29,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589369 2024-12-10T07:26:29,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589369 2024-12-10T07:26:29,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589369) roll requested 2024-12-10T07:26:29,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589377 2024-12-10T07:26:29,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589377 2024-12-10T07:26:29,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589377) roll requested 2024-12-10T07:26:29,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589385 2024-12-10T07:26:29,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589385 2024-12-10T07:26:29,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589385) roll requested 2024-12-10T07:26:29,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589393 2024-12-10T07:26:29,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589393 2024-12-10T07:26:29,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589393) roll requested 2024-12-10T07:26:29,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589401 2024-12-10T07:26:29,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589401 2024-12-10T07:26:29,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589401) roll requested 2024-12-10T07:26:29,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589409 2024-12-10T07:26:29,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589409 2024-12-10T07:26:29,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589409) roll requested 2024-12-10T07:26:29,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589415 2024-12-10T07:26:29,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589415 2024-12-10T07:26:29,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589415) roll requested 2024-12-10T07:26:29,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589422 2024-12-10T07:26:29,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589422 2024-12-10T07:26:29,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589422) roll requested 2024-12-10T07:26:29,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589429 2024-12-10T07:26:29,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589429 2024-12-10T07:26:29,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589429) roll requested 2024-12-10T07:26:29,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589435 2024-12-10T07:26:29,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589435 2024-12-10T07:26:29,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589435) roll requested 2024-12-10T07:26:29,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589441 2024-12-10T07:26:29,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589441 2024-12-10T07:26:29,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589441) roll requested 2024-12-10T07:26:29,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589449 2024-12-10T07:26:29,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589449 2024-12-10T07:26:29,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589449) roll requested 2024-12-10T07:26:29,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589455 2024-12-10T07:26:29,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589455 2024-12-10T07:26:29,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589455) roll requested 2024-12-10T07:26:29,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589460 2024-12-10T07:26:29,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589460 2024-12-10T07:26:29,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589460) roll requested 2024-12-10T07:26:29,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589469 2024-12-10T07:26:29,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589469 2024-12-10T07:26:29,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589469) roll requested 2024-12-10T07:26:29,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589475 2024-12-10T07:26:29,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589475 2024-12-10T07:26:29,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589475) roll requested 2024-12-10T07:26:29,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589480 2024-12-10T07:26:29,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589480 2024-12-10T07:26:29,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589480) roll requested 2024-12-10T07:26:29,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589489 2024-12-10T07:26:29,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589489 2024-12-10T07:26:29,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589489) roll requested 2024-12-10T07:26:29,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589495 2024-12-10T07:26:29,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589495 2024-12-10T07:26:29,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589495) roll requested 2024-12-10T07:26:29,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589501 2024-12-10T07:26:29,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589501 2024-12-10T07:26:29,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589501) roll requested 2024-12-10T07:26:29,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589506 2024-12-10T07:26:29,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589506 2024-12-10T07:26:29,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589506) roll requested 2024-12-10T07:26:29,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589511 2024-12-10T07:26:29,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589511 2024-12-10T07:26:29,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589511) roll requested 2024-12-10T07:26:29,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589516 2024-12-10T07:26:29,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589516 2024-12-10T07:26:29,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589516) roll requested 2024-12-10T07:26:29,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589522 2024-12-10T07:26:29,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589522 2024-12-10T07:26:29,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589522) roll requested 2024-12-10T07:26:29,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589527 2024-12-10T07:26:29,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589527 2024-12-10T07:26:29,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589527) roll requested 2024-12-10T07:26:29,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589531 2024-12-10T07:26:29,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589531 2024-12-10T07:26:29,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589531) roll requested 2024-12-10T07:26:29,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589537 2024-12-10T07:26:29,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589537 2024-12-10T07:26:29,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589537) roll requested 2024-12-10T07:26:29,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589545 2024-12-10T07:26:29,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589545 2024-12-10T07:26:29,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589545) roll requested 2024-12-10T07:26:29,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589550 2024-12-10T07:26:29,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589550 2024-12-10T07:26:29,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589550) roll requested 2024-12-10T07:26:29,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589555 2024-12-10T07:26:29,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589555 2024-12-10T07:26:29,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589555) roll requested 2024-12-10T07:26:29,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589560 2024-12-10T07:26:29,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589560 2024-12-10T07:26:29,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589560) roll requested 2024-12-10T07:26:29,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589565 2024-12-10T07:26:29,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589565 2024-12-10T07:26:29,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589565) roll requested 2024-12-10T07:26:29,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589571 2024-12-10T07:26:29,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589571 2024-12-10T07:26:29,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589571) roll requested 2024-12-10T07:26:29,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589577 2024-12-10T07:26:29,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589577 2024-12-10T07:26:29,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589577) roll requested 2024-12-10T07:26:29,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589582 2024-12-10T07:26:29,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589582 2024-12-10T07:26:29,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589582) roll requested 2024-12-10T07:26:29,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589588 2024-12-10T07:26:29,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589588 2024-12-10T07:26:29,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589588) roll requested 2024-12-10T07:26:29,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589594 2024-12-10T07:26:29,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589594 2024-12-10T07:26:29,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589594) roll requested 2024-12-10T07:26:29,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589600 2024-12-10T07:26:29,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589600 2024-12-10T07:26:29,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589600) roll requested 2024-12-10T07:26:29,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589606 2024-12-10T07:26:29,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589606 2024-12-10T07:26:29,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589606) roll requested 2024-12-10T07:26:29,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589612 2024-12-10T07:26:29,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589612 2024-12-10T07:26:29,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589612) roll requested 2024-12-10T07:26:29,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589619 2024-12-10T07:26:29,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589619 2024-12-10T07:26:29,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589619) roll requested 2024-12-10T07:26:29,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589624 2024-12-10T07:26:29,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589624 2024-12-10T07:26:29,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589624) roll requested 2024-12-10T07:26:29,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589629 2024-12-10T07:26:29,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589629 2024-12-10T07:26:29,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589629) roll requested 2024-12-10T07:26:29,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589633 2024-12-10T07:26:29,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589633 2024-12-10T07:26:29,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589633) roll requested 2024-12-10T07:26:29,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589638 2024-12-10T07:26:29,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589638 2024-12-10T07:26:29,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589638) roll requested 2024-12-10T07:26:29,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589643 2024-12-10T07:26:29,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589643 2024-12-10T07:26:29,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589643) roll requested 2024-12-10T07:26:29,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589649 2024-12-10T07:26:29,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589649 2024-12-10T07:26:29,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589649) roll requested 2024-12-10T07:26:29,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589656 2024-12-10T07:26:29,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589656 2024-12-10T07:26:29,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589656) roll requested 2024-12-10T07:26:29,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589664 2024-12-10T07:26:29,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589664 2024-12-10T07:26:29,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589664) roll requested 2024-12-10T07:26:29,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589671 2024-12-10T07:26:29,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589671 2024-12-10T07:26:29,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589671) roll requested 2024-12-10T07:26:29,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589677 2024-12-10T07:26:29,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589677 2024-12-10T07:26:29,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589677) roll requested 2024-12-10T07:26:29,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589685 2024-12-10T07:26:29,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589685 2024-12-10T07:26:29,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589685) roll requested 2024-12-10T07:26:29,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589693 2024-12-10T07:26:29,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589693 2024-12-10T07:26:29,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589693) roll requested 2024-12-10T07:26:29,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589701 2024-12-10T07:26:29,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589701 2024-12-10T07:26:29,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589701) roll requested 2024-12-10T07:26:29,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589709 2024-12-10T07:26:29,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589709 2024-12-10T07:26:29,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589709) roll requested 2024-12-10T07:26:29,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589718 2024-12-10T07:26:29,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589718 2024-12-10T07:26:29,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589718) roll requested 2024-12-10T07:26:29,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589727 2024-12-10T07:26:29,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589727 2024-12-10T07:26:29,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589727) roll requested 2024-12-10T07:26:29,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589737 2024-12-10T07:26:29,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589737 2024-12-10T07:26:29,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589737) roll requested 2024-12-10T07:26:29,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589744 2024-12-10T07:26:29,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589744 2024-12-10T07:26:29,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589744) roll requested 2024-12-10T07:26:29,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589750 2024-12-10T07:26:29,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589750 2024-12-10T07:26:29,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589750) roll requested 2024-12-10T07:26:29,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589757 2024-12-10T07:26:29,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589757 2024-12-10T07:26:29,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589757) roll requested 2024-12-10T07:26:29,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589766 2024-12-10T07:26:29,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589766 2024-12-10T07:26:29,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589766) roll requested 2024-12-10T07:26:29,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589775 2024-12-10T07:26:29,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589775 2024-12-10T07:26:29,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589775) roll requested 2024-12-10T07:26:29,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589782 2024-12-10T07:26:29,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589782 2024-12-10T07:26:29,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589782) roll requested 2024-12-10T07:26:29,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589791 2024-12-10T07:26:29,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589791 2024-12-10T07:26:29,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589791) roll requested 2024-12-10T07:26:29,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589802 2024-12-10T07:26:29,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589802 2024-12-10T07:26:29,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589802) roll requested 2024-12-10T07:26:29,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589813 2024-12-10T07:26:29,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589813 2024-12-10T07:26:29,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589813) roll requested 2024-12-10T07:26:29,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589824 2024-12-10T07:26:29,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589824 2024-12-10T07:26:29,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589824) roll requested 2024-12-10T07:26:29,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589829 2024-12-10T07:26:29,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589829 2024-12-10T07:26:29,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589829) roll requested 2024-12-10T07:26:29,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589840 2024-12-10T07:26:29,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589840 2024-12-10T07:26:29,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589840) roll requested 2024-12-10T07:26:29,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589845 2024-12-10T07:26:29,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589845 2024-12-10T07:26:29,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589845) roll requested 2024-12-10T07:26:29,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589851 2024-12-10T07:26:29,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589851 2024-12-10T07:26:29,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589851) roll requested 2024-12-10T07:26:29,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589857 2024-12-10T07:26:29,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589857 2024-12-10T07:26:29,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589857) roll requested 2024-12-10T07:26:29,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589862 2024-12-10T07:26:29,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589862 2024-12-10T07:26:29,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589862) roll requested 2024-12-10T07:26:29,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589868 2024-12-10T07:26:29,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589868 2024-12-10T07:26:29,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589868) roll requested 2024-12-10T07:26:29,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589877 2024-12-10T07:26:29,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589877 2024-12-10T07:26:29,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589877) roll requested 2024-12-10T07:26:29,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589883 2024-12-10T07:26:29,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589883 2024-12-10T07:26:29,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589883) roll requested 2024-12-10T07:26:29,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589888 2024-12-10T07:26:29,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589888 2024-12-10T07:26:29,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589888) roll requested 2024-12-10T07:26:29,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589892 2024-12-10T07:26:29,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589892 2024-12-10T07:26:29,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589892) roll requested 2024-12-10T07:26:29,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589898 2024-12-10T07:26:29,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589898 2024-12-10T07:26:29,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589898) roll requested 2024-12-10T07:26:29,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589905 2024-12-10T07:26:29,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589905 2024-12-10T07:26:29,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589905) roll requested 2024-12-10T07:26:29,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589912 2024-12-10T07:26:29,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589912 2024-12-10T07:26:29,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589912) roll requested 2024-12-10T07:26:29,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589920 2024-12-10T07:26:29,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589920 2024-12-10T07:26:29,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589920) roll requested 2024-12-10T07:26:29,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589927 2024-12-10T07:26:29,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589927 2024-12-10T07:26:29,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589927) roll requested 2024-12-10T07:26:29,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589937 2024-12-10T07:26:29,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589937 2024-12-10T07:26:29,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589937) roll requested 2024-12-10T07:26:29,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589946 2024-12-10T07:26:29,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589946 2024-12-10T07:26:29,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589946) roll requested 2024-12-10T07:26:29,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589953 2024-12-10T07:26:29,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589953 2024-12-10T07:26:29,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589953) roll requested 2024-12-10T07:26:29,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589962 2024-12-10T07:26:29,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589962 2024-12-10T07:26:29,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589962) roll requested 2024-12-10T07:26:29,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589971 2024-12-10T07:26:29,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589971 2024-12-10T07:26:29,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589971) roll requested 2024-12-10T07:26:29,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589979 2024-12-10T07:26:29,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589979 2024-12-10T07:26:29,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589979) roll requested 2024-12-10T07:26:29,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589987 2024-12-10T07:26:29,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:29,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589987 2024-12-10T07:26:29,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:29,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:29,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589987) roll requested 2024-12-10T07:26:29,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815589995 2024-12-10T07:26:30,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589995 2024-12-10T07:26:30,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815589995) roll requested 2024-12-10T07:26:30,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590003 2024-12-10T07:26:30,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815589995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590003 2024-12-10T07:26:30,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590003) roll requested 2024-12-10T07:26:30,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590012 2024-12-10T07:26:30,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590012 2024-12-10T07:26:30,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590012) roll requested 2024-12-10T07:26:30,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590020 2024-12-10T07:26:30,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590020 2024-12-10T07:26:30,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590020) roll requested 2024-12-10T07:26:30,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590027 2024-12-10T07:26:30,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590027 2024-12-10T07:26:30,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590027) roll requested 2024-12-10T07:26:30,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590035 2024-12-10T07:26:30,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590035 2024-12-10T07:26:30,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590035) roll requested 2024-12-10T07:26:30,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590044 2024-12-10T07:26:30,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590044 2024-12-10T07:26:30,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590044) roll requested 2024-12-10T07:26:30,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590053 2024-12-10T07:26:30,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590053 2024-12-10T07:26:30,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590053) roll requested 2024-12-10T07:26:30,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590062 2024-12-10T07:26:30,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590062 2024-12-10T07:26:30,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590062) roll requested 2024-12-10T07:26:30,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590072 2024-12-10T07:26:30,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590072 2024-12-10T07:26:30,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590072) roll requested 2024-12-10T07:26:30,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590081 2024-12-10T07:26:30,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590081 2024-12-10T07:26:30,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590081) roll requested 2024-12-10T07:26:30,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590087 2024-12-10T07:26:30,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590087 2024-12-10T07:26:30,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590087) roll requested 2024-12-10T07:26:30,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590150 2024-12-10T07:26:30,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590150 2024-12-10T07:26:30,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590150) roll requested 2024-12-10T07:26:30,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590159 2024-12-10T07:26:30,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590159 2024-12-10T07:26:30,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590159) roll requested 2024-12-10T07:26:30,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590168 2024-12-10T07:26:30,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590168 2024-12-10T07:26:30,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590168) roll requested 2024-12-10T07:26:30,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590176 2024-12-10T07:26:30,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590176 2024-12-10T07:26:30,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590176) roll requested 2024-12-10T07:26:30,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590183 2024-12-10T07:26:30,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590183 2024-12-10T07:26:30,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590183) roll requested 2024-12-10T07:26:30,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590192 2024-12-10T07:26:30,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590192 2024-12-10T07:26:30,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590192) roll requested 2024-12-10T07:26:30,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590200 2024-12-10T07:26:30,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590200 2024-12-10T07:26:30,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590200) roll requested 2024-12-10T07:26:30,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590209 2024-12-10T07:26:30,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590209 2024-12-10T07:26:30,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590209) roll requested 2024-12-10T07:26:30,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590218 2024-12-10T07:26:30,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590218 2024-12-10T07:26:30,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590218) roll requested 2024-12-10T07:26:30,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590227 2024-12-10T07:26:30,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590227 2024-12-10T07:26:30,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590227) roll requested 2024-12-10T07:26:30,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590235 2024-12-10T07:26:30,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590235 2024-12-10T07:26:30,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590235) roll requested 2024-12-10T07:26:30,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590242 2024-12-10T07:26:30,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590242 2024-12-10T07:26:30,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590242) roll requested 2024-12-10T07:26:30,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590248 2024-12-10T07:26:30,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590248 2024-12-10T07:26:30,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590248) roll requested 2024-12-10T07:26:30,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590255 2024-12-10T07:26:30,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590255 2024-12-10T07:26:30,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590255) roll requested 2024-12-10T07:26:30,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590263 2024-12-10T07:26:30,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590263 2024-12-10T07:26:30,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590263) roll requested 2024-12-10T07:26:30,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590271 2024-12-10T07:26:30,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590271 2024-12-10T07:26:30,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590271) roll requested 2024-12-10T07:26:30,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590277 2024-12-10T07:26:30,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590277 2024-12-10T07:26:30,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590277) roll requested 2024-12-10T07:26:30,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590283 2024-12-10T07:26:30,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590283 2024-12-10T07:26:30,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590283) roll requested 2024-12-10T07:26:30,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590288 2024-12-10T07:26:30,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590288 2024-12-10T07:26:30,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590288) roll requested 2024-12-10T07:26:30,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590293 2024-12-10T07:26:30,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590293 2024-12-10T07:26:30,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590293) roll requested 2024-12-10T07:26:30,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590298 2024-12-10T07:26:30,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590298 2024-12-10T07:26:30,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590298) roll requested 2024-12-10T07:26:30,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590305 2024-12-10T07:26:30,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590305 2024-12-10T07:26:30,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590305) roll requested 2024-12-10T07:26:30,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590312 2024-12-10T07:26:30,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590312 2024-12-10T07:26:30,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590312) roll requested 2024-12-10T07:26:30,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590318 2024-12-10T07:26:30,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590318 2024-12-10T07:26:30,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590318) roll requested 2024-12-10T07:26:30,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590325 2024-12-10T07:26:30,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590325 2024-12-10T07:26:30,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590325) roll requested 2024-12-10T07:26:30,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590332 2024-12-10T07:26:30,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590332 2024-12-10T07:26:30,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590332) roll requested 2024-12-10T07:26:30,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590341 2024-12-10T07:26:30,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590341 2024-12-10T07:26:30,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590341) roll requested 2024-12-10T07:26:30,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590349 2024-12-10T07:26:30,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590349 2024-12-10T07:26:30,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590349) roll requested 2024-12-10T07:26:30,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590361 2024-12-10T07:26:30,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590361 2024-12-10T07:26:30,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590361) roll requested 2024-12-10T07:26:30,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590366 2024-12-10T07:26:30,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590366 2024-12-10T07:26:30,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590366) roll requested 2024-12-10T07:26:30,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590371 2024-12-10T07:26:30,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590371 2024-12-10T07:26:30,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590371) roll requested 2024-12-10T07:26:30,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590378 2024-12-10T07:26:30,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590378 2024-12-10T07:26:30,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590378) roll requested 2024-12-10T07:26:30,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590384 2024-12-10T07:26:30,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590384 2024-12-10T07:26:30,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590384) roll requested 2024-12-10T07:26:30,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590392 2024-12-10T07:26:30,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590392 2024-12-10T07:26:30,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590392) roll requested 2024-12-10T07:26:30,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590401 2024-12-10T07:26:30,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590401 2024-12-10T07:26:30,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590401) roll requested 2024-12-10T07:26:30,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590410 2024-12-10T07:26:30,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590410 2024-12-10T07:26:30,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590410) roll requested 2024-12-10T07:26:30,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590420 2024-12-10T07:26:30,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590420 2024-12-10T07:26:30,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590420) roll requested 2024-12-10T07:26:30,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590427 2024-12-10T07:26:30,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590427 2024-12-10T07:26:30,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590427) roll requested 2024-12-10T07:26:30,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590433 2024-12-10T07:26:30,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590433 2024-12-10T07:26:30,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590433) roll requested 2024-12-10T07:26:30,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590441 2024-12-10T07:26:30,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590441 2024-12-10T07:26:30,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590441) roll requested 2024-12-10T07:26:30,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590450 2024-12-10T07:26:30,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590450 2024-12-10T07:26:30,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590450) roll requested 2024-12-10T07:26:30,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590460 2024-12-10T07:26:30,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590460 2024-12-10T07:26:30,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590460) roll requested 2024-12-10T07:26:30,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590470 2024-12-10T07:26:30,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590470 2024-12-10T07:26:30,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590470) roll requested 2024-12-10T07:26:30,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590477 2024-12-10T07:26:30,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590477 2024-12-10T07:26:30,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590477) roll requested 2024-12-10T07:26:30,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590486 2024-12-10T07:26:30,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590486 2024-12-10T07:26:30,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590486) roll requested 2024-12-10T07:26:30,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590495 2024-12-10T07:26:30,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590495 2024-12-10T07:26:30,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590495) roll requested 2024-12-10T07:26:30,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590502 2024-12-10T07:26:30,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590502 2024-12-10T07:26:30,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590502) roll requested 2024-12-10T07:26:30,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590509 2024-12-10T07:26:30,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590509 2024-12-10T07:26:30,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590509) roll requested 2024-12-10T07:26:30,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590517 2024-12-10T07:26:30,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590517 2024-12-10T07:26:30,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590517) roll requested 2024-12-10T07:26:30,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590526 2024-12-10T07:26:30,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590526 2024-12-10T07:26:30,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590526) roll requested 2024-12-10T07:26:30,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590535 2024-12-10T07:26:30,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590535 2024-12-10T07:26:30,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590535) roll requested 2024-12-10T07:26:30,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590544 2024-12-10T07:26:30,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590544 2024-12-10T07:26:30,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590544) roll requested 2024-12-10T07:26:30,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590552 2024-12-10T07:26:30,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590552 2024-12-10T07:26:30,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590552) roll requested 2024-12-10T07:26:30,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590559 2024-12-10T07:26:30,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590559 2024-12-10T07:26:30,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590559) roll requested 2024-12-10T07:26:30,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590566 2024-12-10T07:26:30,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590566 2024-12-10T07:26:30,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590566) roll requested 2024-12-10T07:26:30,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590574 2024-12-10T07:26:30,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590574 2024-12-10T07:26:30,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590574) roll requested 2024-12-10T07:26:30,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590581 2024-12-10T07:26:30,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590581 2024-12-10T07:26:30,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590581) roll requested 2024-12-10T07:26:30,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590589 2024-12-10T07:26:30,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590589 2024-12-10T07:26:30,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590589) roll requested 2024-12-10T07:26:30,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590597 2024-12-10T07:26:30,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590597 2024-12-10T07:26:30,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590597) roll requested 2024-12-10T07:26:30,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590604 2024-12-10T07:26:30,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590604 2024-12-10T07:26:30,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590604) roll requested 2024-12-10T07:26:30,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590613 2024-12-10T07:26:30,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590613 2024-12-10T07:26:30,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590613) roll requested 2024-12-10T07:26:30,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590622 2024-12-10T07:26:30,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590622 2024-12-10T07:26:30,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590622) roll requested 2024-12-10T07:26:30,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590628 2024-12-10T07:26:30,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590628 2024-12-10T07:26:30,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590628) roll requested 2024-12-10T07:26:30,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590633 2024-12-10T07:26:30,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590633 2024-12-10T07:26:30,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590633) roll requested 2024-12-10T07:26:30,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590639 2024-12-10T07:26:30,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590639 2024-12-10T07:26:30,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590639) roll requested 2024-12-10T07:26:30,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590644 2024-12-10T07:26:30,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590644 2024-12-10T07:26:30,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590644) roll requested 2024-12-10T07:26:30,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590649 2024-12-10T07:26:30,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590649 2024-12-10T07:26:30,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590649) roll requested 2024-12-10T07:26:30,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590655 2024-12-10T07:26:30,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590655 2024-12-10T07:26:30,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590655) roll requested 2024-12-10T07:26:30,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590662 2024-12-10T07:26:30,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590662 2024-12-10T07:26:30,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590662) roll requested 2024-12-10T07:26:30,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590668 2024-12-10T07:26:30,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590668 2024-12-10T07:26:30,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590668) roll requested 2024-12-10T07:26:30,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590674 2024-12-10T07:26:30,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590674 2024-12-10T07:26:30,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590674) roll requested 2024-12-10T07:26:30,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590680 2024-12-10T07:26:30,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590680 2024-12-10T07:26:30,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590680) roll requested 2024-12-10T07:26:30,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590685 2024-12-10T07:26:30,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590685 2024-12-10T07:26:30,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590685) roll requested 2024-12-10T07:26:30,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590692 2024-12-10T07:26:30,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590692 2024-12-10T07:26:30,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590692) roll requested 2024-12-10T07:26:30,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590697 2024-12-10T07:26:30,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590697 2024-12-10T07:26:30,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590697) roll requested 2024-12-10T07:26:30,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590703 2024-12-10T07:26:30,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590703 2024-12-10T07:26:30,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590703) roll requested 2024-12-10T07:26:30,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590709 2024-12-10T07:26:30,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590709 2024-12-10T07:26:30,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590709) roll requested 2024-12-10T07:26:30,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590715 2024-12-10T07:26:30,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590715 2024-12-10T07:26:30,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590715) roll requested 2024-12-10T07:26:30,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590721 2024-12-10T07:26:30,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590721 2024-12-10T07:26:30,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590721) roll requested 2024-12-10T07:26:30,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590726 2024-12-10T07:26:30,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590726 2024-12-10T07:26:30,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590726) roll requested 2024-12-10T07:26:30,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590732 2024-12-10T07:26:30,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590732 2024-12-10T07:26:30,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590732) roll requested 2024-12-10T07:26:30,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590738 2024-12-10T07:26:30,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590738 2024-12-10T07:26:30,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590738) roll requested 2024-12-10T07:26:30,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590744 2024-12-10T07:26:30,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590744 2024-12-10T07:26:30,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590744) roll requested 2024-12-10T07:26:30,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590750 2024-12-10T07:26:30,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590750 2024-12-10T07:26:30,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590750) roll requested 2024-12-10T07:26:30,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590756 2024-12-10T07:26:30,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590756 2024-12-10T07:26:30,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590756) roll requested 2024-12-10T07:26:30,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590762 2024-12-10T07:26:30,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590762 2024-12-10T07:26:30,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590762) roll requested 2024-12-10T07:26:30,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590768 2024-12-10T07:26:30,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590768 2024-12-10T07:26:30,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590768) roll requested 2024-12-10T07:26:30,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590774 2024-12-10T07:26:30,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590774 2024-12-10T07:26:30,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590774) roll requested 2024-12-10T07:26:30,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590781 2024-12-10T07:26:30,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590781 2024-12-10T07:26:30,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590781) roll requested 2024-12-10T07:26:30,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590786 2024-12-10T07:26:30,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590786 2024-12-10T07:26:30,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590786) roll requested 2024-12-10T07:26:30,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590792 2024-12-10T07:26:30,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590792 2024-12-10T07:26:30,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590792) roll requested 2024-12-10T07:26:30,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590798 2024-12-10T07:26:30,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590798 2024-12-10T07:26:30,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590798) roll requested 2024-12-10T07:26:30,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590803 2024-12-10T07:26:30,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590803 2024-12-10T07:26:30,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590803) roll requested 2024-12-10T07:26:30,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590809 2024-12-10T07:26:30,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590809 2024-12-10T07:26:30,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590809) roll requested 2024-12-10T07:26:30,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590815 2024-12-10T07:26:30,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590815 2024-12-10T07:26:30,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590815) roll requested 2024-12-10T07:26:30,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590821 2024-12-10T07:26:30,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590821 2024-12-10T07:26:30,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590821) roll requested 2024-12-10T07:26:30,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590826 2024-12-10T07:26:30,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590826 2024-12-10T07:26:30,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590826) roll requested 2024-12-10T07:26:30,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590832 2024-12-10T07:26:30,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590832 2024-12-10T07:26:30,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590832) roll requested 2024-12-10T07:26:30,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590838 2024-12-10T07:26:30,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590838 2024-12-10T07:26:30,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590838) roll requested 2024-12-10T07:26:30,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590843 2024-12-10T07:26:30,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590843 2024-12-10T07:26:30,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590843) roll requested 2024-12-10T07:26:30,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590848 2024-12-10T07:26:30,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590848 2024-12-10T07:26:30,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590848) roll requested 2024-12-10T07:26:30,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590853 2024-12-10T07:26:30,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590853 2024-12-10T07:26:30,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590853) roll requested 2024-12-10T07:26:30,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590858 2024-12-10T07:26:30,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590858 2024-12-10T07:26:30,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590858) roll requested 2024-12-10T07:26:30,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590864 2024-12-10T07:26:30,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590864 2024-12-10T07:26:30,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590864) roll requested 2024-12-10T07:26:30,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590871 2024-12-10T07:26:30,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590871 2024-12-10T07:26:30,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590871) roll requested 2024-12-10T07:26:30,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590878 2024-12-10T07:26:30,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590878 2024-12-10T07:26:30,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590878) roll requested 2024-12-10T07:26:30,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590886 2024-12-10T07:26:30,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590886 2024-12-10T07:26:30,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590886) roll requested 2024-12-10T07:26:30,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590893 2024-12-10T07:26:30,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590893 2024-12-10T07:26:30,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590893) roll requested 2024-12-10T07:26:30,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590903 2024-12-10T07:26:30,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590903 2024-12-10T07:26:30,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590903) roll requested 2024-12-10T07:26:30,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590912 2024-12-10T07:26:30,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590912 2024-12-10T07:26:30,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590912) roll requested 2024-12-10T07:26:30,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590920 2024-12-10T07:26:30,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590920 2024-12-10T07:26:30,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590920) roll requested 2024-12-10T07:26:30,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590926 2024-12-10T07:26:30,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590926 2024-12-10T07:26:30,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590926) roll requested 2024-12-10T07:26:30,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590933 2024-12-10T07:26:30,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590933 2024-12-10T07:26:30,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590933) roll requested 2024-12-10T07:26:30,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590941 2024-12-10T07:26:30,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590941 2024-12-10T07:26:30,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590941) roll requested 2024-12-10T07:26:30,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590947 2024-12-10T07:26:30,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590947 2024-12-10T07:26:30,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590947) roll requested 2024-12-10T07:26:30,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590953 2024-12-10T07:26:30,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590953 2024-12-10T07:26:30,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590953) roll requested 2024-12-10T07:26:30,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590959 2024-12-10T07:26:30,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590959 2024-12-10T07:26:30,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590959) roll requested 2024-12-10T07:26:30,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590966 2024-12-10T07:26:30,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590966 2024-12-10T07:26:30,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590966) roll requested 2024-12-10T07:26:30,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590974 2024-12-10T07:26:30,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590974 2024-12-10T07:26:30,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590974) roll requested 2024-12-10T07:26:30,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590981 2024-12-10T07:26:30,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590981 2024-12-10T07:26:30,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590981) roll requested 2024-12-10T07:26:30,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590988 2024-12-10T07:26:30,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:30,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590988 2024-12-10T07:26:30,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:30,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:30,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590988) roll requested 2024-12-10T07:26:30,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815590996 2024-12-10T07:26:31,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590996 2024-12-10T07:26:31,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815590996) roll requested 2024-12-10T07:26:31,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591013 2024-12-10T07:26:31,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815590996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591013 2024-12-10T07:26:31,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591013) roll requested 2024-12-10T07:26:31,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591018 2024-12-10T07:26:31,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591018 2024-12-10T07:26:31,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591018) roll requested 2024-12-10T07:26:31,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591023 2024-12-10T07:26:31,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591023 2024-12-10T07:26:31,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591023) roll requested 2024-12-10T07:26:31,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591029 2024-12-10T07:26:31,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591029 2024-12-10T07:26:31,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591029) roll requested 2024-12-10T07:26:31,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591035 2024-12-10T07:26:31,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591035 2024-12-10T07:26:31,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591035) roll requested 2024-12-10T07:26:31,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591042 2024-12-10T07:26:31,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591042 2024-12-10T07:26:31,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591042) roll requested 2024-12-10T07:26:31,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591049 2024-12-10T07:26:31,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591049 2024-12-10T07:26:31,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591049) roll requested 2024-12-10T07:26:31,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591054 2024-12-10T07:26:31,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591054 2024-12-10T07:26:31,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591054) roll requested 2024-12-10T07:26:31,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591060 2024-12-10T07:26:31,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591060 2024-12-10T07:26:31,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591060) roll requested 2024-12-10T07:26:31,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591066 2024-12-10T07:26:31,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591066 2024-12-10T07:26:31,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591066) roll requested 2024-12-10T07:26:31,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591070 2024-12-10T07:26:31,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591070 2024-12-10T07:26:31,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591070) roll requested 2024-12-10T07:26:31,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591076 2024-12-10T07:26:31,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591076 2024-12-10T07:26:31,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591076) roll requested 2024-12-10T07:26:31,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591082 2024-12-10T07:26:31,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591082 2024-12-10T07:26:31,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591082) roll requested 2024-12-10T07:26:31,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591089 2024-12-10T07:26:31,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591089 2024-12-10T07:26:31,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591089) roll requested 2024-12-10T07:26:31,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591097 2024-12-10T07:26:31,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591097 2024-12-10T07:26:31,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591097) roll requested 2024-12-10T07:26:31,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591102 2024-12-10T07:26:31,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591102 2024-12-10T07:26:31,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591102) roll requested 2024-12-10T07:26:31,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591107 2024-12-10T07:26:31,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591107 2024-12-10T07:26:31,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591107) roll requested 2024-12-10T07:26:31,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591114 2024-12-10T07:26:31,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591114 2024-12-10T07:26:31,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591114) roll requested 2024-12-10T07:26:31,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591120 2024-12-10T07:26:31,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591120 2024-12-10T07:26:31,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591120) roll requested 2024-12-10T07:26:31,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591127 2024-12-10T07:26:31,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591127 2024-12-10T07:26:31,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591127) roll requested 2024-12-10T07:26:31,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591134 2024-12-10T07:26:31,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591134 2024-12-10T07:26:31,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591134) roll requested 2024-12-10T07:26:31,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591142 2024-12-10T07:26:31,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591142 2024-12-10T07:26:31,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591142) roll requested 2024-12-10T07:26:31,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591150 2024-12-10T07:26:31,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591150 2024-12-10T07:26:31,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591150) roll requested 2024-12-10T07:26:31,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591159 2024-12-10T07:26:31,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591159 2024-12-10T07:26:31,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591159) roll requested 2024-12-10T07:26:31,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591168 2024-12-10T07:26:31,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591168 2024-12-10T07:26:31,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591168) roll requested 2024-12-10T07:26:31,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591175 2024-12-10T07:26:31,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591175 2024-12-10T07:26:31,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591175) roll requested 2024-12-10T07:26:31,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591182 2024-12-10T07:26:31,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591182 2024-12-10T07:26:31,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591182) roll requested 2024-12-10T07:26:31,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591191 2024-12-10T07:26:31,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591191 2024-12-10T07:26:31,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591191) roll requested 2024-12-10T07:26:31,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591200 2024-12-10T07:26:31,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591200 2024-12-10T07:26:31,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591200) roll requested 2024-12-10T07:26:31,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591210 2024-12-10T07:26:31,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591210 2024-12-10T07:26:31,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591210) roll requested 2024-12-10T07:26:31,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591220 2024-12-10T07:26:31,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591220 2024-12-10T07:26:31,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591220) roll requested 2024-12-10T07:26:31,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591230 2024-12-10T07:26:31,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591230 2024-12-10T07:26:31,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591230) roll requested 2024-12-10T07:26:31,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591240 2024-12-10T07:26:31,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591240 2024-12-10T07:26:31,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591240) roll requested 2024-12-10T07:26:31,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591249 2024-12-10T07:26:31,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591249 2024-12-10T07:26:31,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591249) roll requested 2024-12-10T07:26:31,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591258 2024-12-10T07:26:31,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591258 2024-12-10T07:26:31,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591258) roll requested 2024-12-10T07:26:31,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591267 2024-12-10T07:26:31,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591267 2024-12-10T07:26:31,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591267) roll requested 2024-12-10T07:26:31,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591275 2024-12-10T07:26:31,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591275 2024-12-10T07:26:31,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591275) roll requested 2024-12-10T07:26:31,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591283 2024-12-10T07:26:31,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591283 2024-12-10T07:26:31,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591283) roll requested 2024-12-10T07:26:31,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591290 2024-12-10T07:26:31,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591290 2024-12-10T07:26:31,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591290) roll requested 2024-12-10T07:26:31,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591296 2024-12-10T07:26:31,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591296 2024-12-10T07:26:31,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591296) roll requested 2024-12-10T07:26:31,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591303 2024-12-10T07:26:31,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591303 2024-12-10T07:26:31,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591303) roll requested 2024-12-10T07:26:31,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591311 2024-12-10T07:26:31,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591311 2024-12-10T07:26:31,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591311) roll requested 2024-12-10T07:26:31,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591320 2024-12-10T07:26:31,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591320 2024-12-10T07:26:31,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591320) roll requested 2024-12-10T07:26:31,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591328 2024-12-10T07:26:31,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591328 2024-12-10T07:26:31,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591328) roll requested 2024-12-10T07:26:31,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591337 2024-12-10T07:26:31,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591337 2024-12-10T07:26:31,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591337) roll requested 2024-12-10T07:26:31,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591344 2024-12-10T07:26:31,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591344 2024-12-10T07:26:31,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591344) roll requested 2024-12-10T07:26:31,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591352 2024-12-10T07:26:31,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591352 2024-12-10T07:26:31,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591352) roll requested 2024-12-10T07:26:31,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591361 2024-12-10T07:26:31,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591361 2024-12-10T07:26:31,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591361) roll requested 2024-12-10T07:26:31,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591371 2024-12-10T07:26:31,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591371 2024-12-10T07:26:31,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591371) roll requested 2024-12-10T07:26:31,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591379 2024-12-10T07:26:31,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591379 2024-12-10T07:26:31,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591379) roll requested 2024-12-10T07:26:31,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591387 2024-12-10T07:26:31,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591387 2024-12-10T07:26:31,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591387) roll requested 2024-12-10T07:26:31,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591395 2024-12-10T07:26:31,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591395 2024-12-10T07:26:31,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591395) roll requested 2024-12-10T07:26:31,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591403 2024-12-10T07:26:31,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591403 2024-12-10T07:26:31,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591403) roll requested 2024-12-10T07:26:31,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591411 2024-12-10T07:26:31,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591411 2024-12-10T07:26:31,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591411) roll requested 2024-12-10T07:26:31,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591421 2024-12-10T07:26:31,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591421 2024-12-10T07:26:31,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591421) roll requested 2024-12-10T07:26:31,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591431 2024-12-10T07:26:31,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591431 2024-12-10T07:26:31,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591431) roll requested 2024-12-10T07:26:31,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591441 2024-12-10T07:26:31,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591441 2024-12-10T07:26:31,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591441) roll requested 2024-12-10T07:26:31,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591450 2024-12-10T07:26:31,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591450 2024-12-10T07:26:31,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591450) roll requested 2024-12-10T07:26:31,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591460 2024-12-10T07:26:31,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591460 2024-12-10T07:26:31,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591460) roll requested 2024-12-10T07:26:31,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591468 2024-12-10T07:26:31,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591468 2024-12-10T07:26:31,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591468) roll requested 2024-12-10T07:26:31,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591475 2024-12-10T07:26:31,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591475 2024-12-10T07:26:31,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591475) roll requested 2024-12-10T07:26:31,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591483 2024-12-10T07:26:31,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591483 2024-12-10T07:26:31,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591483) roll requested 2024-12-10T07:26:31,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591492 2024-12-10T07:26:31,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591492 2024-12-10T07:26:31,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591492) roll requested 2024-12-10T07:26:31,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591501 2024-12-10T07:26:31,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591501 2024-12-10T07:26:31,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591501) roll requested 2024-12-10T07:26:31,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591508 2024-12-10T07:26:31,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591508 2024-12-10T07:26:31,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591508) roll requested 2024-12-10T07:26:31,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591515 2024-12-10T07:26:31,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591515 2024-12-10T07:26:31,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591515) roll requested 2024-12-10T07:26:31,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591522 2024-12-10T07:26:31,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591522 2024-12-10T07:26:31,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591522) roll requested 2024-12-10T07:26:31,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591529 2024-12-10T07:26:31,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591529 2024-12-10T07:26:31,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591529) roll requested 2024-12-10T07:26:31,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591538 2024-12-10T07:26:31,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591538 2024-12-10T07:26:31,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591538) roll requested 2024-12-10T07:26:31,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591545 2024-12-10T07:26:31,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591545 2024-12-10T07:26:31,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591545) roll requested 2024-12-10T07:26:31,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591552 2024-12-10T07:26:31,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591552 2024-12-10T07:26:31,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591552) roll requested 2024-12-10T07:26:31,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591558 2024-12-10T07:26:31,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591558 2024-12-10T07:26:31,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591558) roll requested 2024-12-10T07:26:31,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591565 2024-12-10T07:26:31,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591565 2024-12-10T07:26:31,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591565) roll requested 2024-12-10T07:26:31,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591573 2024-12-10T07:26:31,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591573 2024-12-10T07:26:31,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591573) roll requested 2024-12-10T07:26:31,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591580 2024-12-10T07:26:31,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591580 2024-12-10T07:26:31,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591580) roll requested 2024-12-10T07:26:31,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591589 2024-12-10T07:26:31,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591589 2024-12-10T07:26:31,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591589) roll requested 2024-12-10T07:26:31,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591596 2024-12-10T07:26:31,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591596 2024-12-10T07:26:31,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591596) roll requested 2024-12-10T07:26:31,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591601 2024-12-10T07:26:31,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591601 2024-12-10T07:26:31,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591601) roll requested 2024-12-10T07:26:31,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591606 2024-12-10T07:26:31,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591606 2024-12-10T07:26:31,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591606) roll requested 2024-12-10T07:26:31,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591612 2024-12-10T07:26:31,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591612 2024-12-10T07:26:31,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591612) roll requested 2024-12-10T07:26:31,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591618 2024-12-10T07:26:31,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591618 2024-12-10T07:26:31,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591618) roll requested 2024-12-10T07:26:31,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591626 2024-12-10T07:26:31,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591626 2024-12-10T07:26:31,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591626) roll requested 2024-12-10T07:26:31,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591634 2024-12-10T07:26:31,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591634 2024-12-10T07:26:31,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591634) roll requested 2024-12-10T07:26:31,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591642 2024-12-10T07:26:31,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591642 2024-12-10T07:26:31,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591642) roll requested 2024-12-10T07:26:31,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591650 2024-12-10T07:26:31,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591650 2024-12-10T07:26:31,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591650) roll requested 2024-12-10T07:26:31,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591658 2024-12-10T07:26:31,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591658 2024-12-10T07:26:31,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591658) roll requested 2024-12-10T07:26:31,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591666 2024-12-10T07:26:31,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591666 2024-12-10T07:26:31,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591666) roll requested 2024-12-10T07:26:31,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591674 2024-12-10T07:26:31,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591674 2024-12-10T07:26:31,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591674) roll requested 2024-12-10T07:26:31,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591680 2024-12-10T07:26:31,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591680 2024-12-10T07:26:31,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591680) roll requested 2024-12-10T07:26:31,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591688 2024-12-10T07:26:31,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591688 2024-12-10T07:26:31,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591688) roll requested 2024-12-10T07:26:31,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591697 2024-12-10T07:26:31,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591697 2024-12-10T07:26:31,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591697) roll requested 2024-12-10T07:26:31,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591706 2024-12-10T07:26:31,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591706 2024-12-10T07:26:31,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591706) roll requested 2024-12-10T07:26:31,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591714 2024-12-10T07:26:31,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591714 2024-12-10T07:26:31,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591714) roll requested 2024-12-10T07:26:31,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591720 2024-12-10T07:26:31,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591720 2024-12-10T07:26:31,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591720) roll requested 2024-12-10T07:26:31,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591726 2024-12-10T07:26:31,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591726 2024-12-10T07:26:31,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591726) roll requested 2024-12-10T07:26:31,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591732 2024-12-10T07:26:31,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591732 2024-12-10T07:26:31,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591732) roll requested 2024-12-10T07:26:31,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591739 2024-12-10T07:26:31,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591739 2024-12-10T07:26:31,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591739) roll requested 2024-12-10T07:26:31,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591748 2024-12-10T07:26:31,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591748 2024-12-10T07:26:31,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591748) roll requested 2024-12-10T07:26:31,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591756 2024-12-10T07:26:31,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591756 2024-12-10T07:26:31,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591756) roll requested 2024-12-10T07:26:31,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591765 2024-12-10T07:26:31,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591765 2024-12-10T07:26:31,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591765) roll requested 2024-12-10T07:26:31,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591774 2024-12-10T07:26:31,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591774 2024-12-10T07:26:31,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591774) roll requested 2024-12-10T07:26:31,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591779 2024-12-10T07:26:31,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591779 2024-12-10T07:26:31,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591779) roll requested 2024-12-10T07:26:31,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591785 2024-12-10T07:26:31,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591785 2024-12-10T07:26:31,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591785) roll requested 2024-12-10T07:26:31,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591793 2024-12-10T07:26:31,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591793 2024-12-10T07:26:31,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591793) roll requested 2024-12-10T07:26:31,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591801 2024-12-10T07:26:31,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591801 2024-12-10T07:26:31,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591801) roll requested 2024-12-10T07:26:31,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591811 2024-12-10T07:26:31,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591811 2024-12-10T07:26:31,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591811) roll requested 2024-12-10T07:26:31,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591821 2024-12-10T07:26:31,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591821 2024-12-10T07:26:31,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591821) roll requested 2024-12-10T07:26:31,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591829 2024-12-10T07:26:31,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591829 2024-12-10T07:26:31,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591829) roll requested 2024-12-10T07:26:31,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591838 2024-12-10T07:26:31,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591838 2024-12-10T07:26:31,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591838) roll requested 2024-12-10T07:26:31,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591847 2024-12-10T07:26:31,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591847 2024-12-10T07:26:31,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591847) roll requested 2024-12-10T07:26:31,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591856 2024-12-10T07:26:31,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591856 2024-12-10T07:26:31,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591856) roll requested 2024-12-10T07:26:31,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591866 2024-12-10T07:26:31,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591866 2024-12-10T07:26:31,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591866) roll requested 2024-12-10T07:26:31,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591874 2024-12-10T07:26:31,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591874 2024-12-10T07:26:31,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591874) roll requested 2024-12-10T07:26:31,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591885 2024-12-10T07:26:31,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591885 2024-12-10T07:26:31,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591885) roll requested 2024-12-10T07:26:31,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591896 2024-12-10T07:26:31,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591896 2024-12-10T07:26:31,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591896) roll requested 2024-12-10T07:26:31,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591901 2024-12-10T07:26:31,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591901 2024-12-10T07:26:31,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591901) roll requested 2024-12-10T07:26:31,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591905 2024-12-10T07:26:31,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591905 2024-12-10T07:26:31,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591905) roll requested 2024-12-10T07:26:31,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591910 2024-12-10T07:26:31,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591910 2024-12-10T07:26:31,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591910) roll requested 2024-12-10T07:26:31,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591918 2024-12-10T07:26:31,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591918 2024-12-10T07:26:31,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591918) roll requested 2024-12-10T07:26:31,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591924 2024-12-10T07:26:31,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591924 2024-12-10T07:26:31,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591924) roll requested 2024-12-10T07:26:31,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591932 2024-12-10T07:26:31,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591932 2024-12-10T07:26:31,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591932) roll requested 2024-12-10T07:26:31,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591940 2024-12-10T07:26:31,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591940 2024-12-10T07:26:31,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591940) roll requested 2024-12-10T07:26:31,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591948 2024-12-10T07:26:31,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591948 2024-12-10T07:26:31,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591948) roll requested 2024-12-10T07:26:31,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591955 2024-12-10T07:26:31,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591955 2024-12-10T07:26:31,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591955) roll requested 2024-12-10T07:26:31,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591962 2024-12-10T07:26:31,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591962 2024-12-10T07:26:31,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591962) roll requested 2024-12-10T07:26:31,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591973 2024-12-10T07:26:31,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591973 2024-12-10T07:26:31,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591973) roll requested 2024-12-10T07:26:31,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591982 2024-12-10T07:26:31,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591982 2024-12-10T07:26:31,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591982) roll requested 2024-12-10T07:26:31,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591987 2024-12-10T07:26:31,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591987 2024-12-10T07:26:31,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591987) roll requested 2024-12-10T07:26:31,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591992 2024-12-10T07:26:31,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:31,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591992 2024-12-10T07:26:31,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:31,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:31,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591992) roll requested 2024-12-10T07:26:31,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815591998 2024-12-10T07:26:32,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591998 2024-12-10T07:26:32,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815591998) roll requested 2024-12-10T07:26:32,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592005 2024-12-10T07:26:32,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815591998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592005 2024-12-10T07:26:32,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592005) roll requested 2024-12-10T07:26:32,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592012 2024-12-10T07:26:32,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592012 2024-12-10T07:26:32,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592012) roll requested 2024-12-10T07:26:32,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592020 2024-12-10T07:26:32,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592020 2024-12-10T07:26:32,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592020) roll requested 2024-12-10T07:26:32,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592027 2024-12-10T07:26:32,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592027 2024-12-10T07:26:32,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592027) roll requested 2024-12-10T07:26:32,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592032 2024-12-10T07:26:32,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592032 2024-12-10T07:26:32,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592032) roll requested 2024-12-10T07:26:32,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592039 2024-12-10T07:26:32,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592039 2024-12-10T07:26:32,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592039) roll requested 2024-12-10T07:26:32,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592047 2024-12-10T07:26:32,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592047 2024-12-10T07:26:32,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592047) roll requested 2024-12-10T07:26:32,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592056 2024-12-10T07:26:32,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592056 2024-12-10T07:26:32,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592056) roll requested 2024-12-10T07:26:32,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592065 2024-12-10T07:26:32,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592065 2024-12-10T07:26:32,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592065) roll requested 2024-12-10T07:26:32,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592074 2024-12-10T07:26:32,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592074 2024-12-10T07:26:32,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592074) roll requested 2024-12-10T07:26:32,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592087 2024-12-10T07:26:32,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592087 2024-12-10T07:26:32,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592087) roll requested 2024-12-10T07:26:32,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592095 2024-12-10T07:26:32,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592095 2024-12-10T07:26:32,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592095) roll requested 2024-12-10T07:26:32,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592099 2024-12-10T07:26:32,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592099 2024-12-10T07:26:32,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592099) roll requested 2024-12-10T07:26:32,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592104 2024-12-10T07:26:32,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592104 2024-12-10T07:26:32,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592104) roll requested 2024-12-10T07:26:32,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592109 2024-12-10T07:26:32,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592109 2024-12-10T07:26:32,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592109) roll requested 2024-12-10T07:26:32,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592116 2024-12-10T07:26:32,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592116 2024-12-10T07:26:32,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592116) roll requested 2024-12-10T07:26:32,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592123 2024-12-10T07:26:32,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592123 2024-12-10T07:26:32,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592123) roll requested 2024-12-10T07:26:32,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592130 2024-12-10T07:26:32,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592130 2024-12-10T07:26:32,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592130) roll requested 2024-12-10T07:26:32,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592137 2024-12-10T07:26:32,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592137 2024-12-10T07:26:32,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592137) roll requested 2024-12-10T07:26:32,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592143 2024-12-10T07:26:32,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592143 2024-12-10T07:26:32,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592143) roll requested 2024-12-10T07:26:32,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592150 2024-12-10T07:26:32,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592150 2024-12-10T07:26:32,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592150) roll requested 2024-12-10T07:26:32,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592158 2024-12-10T07:26:32,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592158 2024-12-10T07:26:32,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592158) roll requested 2024-12-10T07:26:32,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592166 2024-12-10T07:26:32,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592166 2024-12-10T07:26:32,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592166) roll requested 2024-12-10T07:26:32,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592173 2024-12-10T07:26:32,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592173 2024-12-10T07:26:32,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592173) roll requested 2024-12-10T07:26:32,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592181 2024-12-10T07:26:32,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592181 2024-12-10T07:26:32,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592181) roll requested 2024-12-10T07:26:32,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592191 2024-12-10T07:26:32,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592191 2024-12-10T07:26:32,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592191) roll requested 2024-12-10T07:26:32,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592197 2024-12-10T07:26:32,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592197 2024-12-10T07:26:32,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592197) roll requested 2024-12-10T07:26:32,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592202 2024-12-10T07:26:32,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592202 2024-12-10T07:26:32,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592202) roll requested 2024-12-10T07:26:32,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592209 2024-12-10T07:26:32,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592209 2024-12-10T07:26:32,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592209) roll requested 2024-12-10T07:26:32,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592221 2024-12-10T07:26:32,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592221 2024-12-10T07:26:32,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592221) roll requested 2024-12-10T07:26:32,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592227 2024-12-10T07:26:32,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592227 2024-12-10T07:26:32,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592227) roll requested 2024-12-10T07:26:32,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592232 2024-12-10T07:26:32,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592232 2024-12-10T07:26:32,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592232) roll requested 2024-12-10T07:26:32,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592236 2024-12-10T07:26:32,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592236 2024-12-10T07:26:32,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592236) roll requested 2024-12-10T07:26:32,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592243 2024-12-10T07:26:32,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592243 2024-12-10T07:26:32,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592243) roll requested 2024-12-10T07:26:32,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592249 2024-12-10T07:26:32,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592249 2024-12-10T07:26:32,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592249) roll requested 2024-12-10T07:26:32,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592258 2024-12-10T07:26:32,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592258 2024-12-10T07:26:32,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592258) roll requested 2024-12-10T07:26:32,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592265 2024-12-10T07:26:32,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592265 2024-12-10T07:26:32,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592265) roll requested 2024-12-10T07:26:32,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592273 2024-12-10T07:26:32,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592273 2024-12-10T07:26:32,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592273) roll requested 2024-12-10T07:26:32,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592279 2024-12-10T07:26:32,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592279 2024-12-10T07:26:32,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592279) roll requested 2024-12-10T07:26:32,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592287 2024-12-10T07:26:32,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592287 2024-12-10T07:26:32,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592287) roll requested 2024-12-10T07:26:32,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592294 2024-12-10T07:26:32,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592294 2024-12-10T07:26:32,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592294) roll requested 2024-12-10T07:26:32,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592300 2024-12-10T07:26:32,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592300 2024-12-10T07:26:32,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592300) roll requested 2024-12-10T07:26:32,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592306 2024-12-10T07:26:32,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592306 2024-12-10T07:26:32,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592306) roll requested 2024-12-10T07:26:32,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592314 2024-12-10T07:26:32,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592314 2024-12-10T07:26:32,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592314) roll requested 2024-12-10T07:26:32,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592322 2024-12-10T07:26:32,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592322 2024-12-10T07:26:32,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592322) roll requested 2024-12-10T07:26:32,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592331 2024-12-10T07:26:32,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592331 2024-12-10T07:26:32,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592331) roll requested 2024-12-10T07:26:32,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592341 2024-12-10T07:26:32,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592341 2024-12-10T07:26:32,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592341) roll requested 2024-12-10T07:26:32,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592346 2024-12-10T07:26:32,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592346 2024-12-10T07:26:32,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592346) roll requested 2024-12-10T07:26:32,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592351 2024-12-10T07:26:32,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592351 2024-12-10T07:26:32,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592351) roll requested 2024-12-10T07:26:32,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592356 2024-12-10T07:26:32,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592356 2024-12-10T07:26:32,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592356) roll requested 2024-12-10T07:26:32,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592362 2024-12-10T07:26:32,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592362 2024-12-10T07:26:32,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592362) roll requested 2024-12-10T07:26:32,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592369 2024-12-10T07:26:32,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592369 2024-12-10T07:26:32,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592369) roll requested 2024-12-10T07:26:32,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592376 2024-12-10T07:26:32,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592376 2024-12-10T07:26:32,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592376) roll requested 2024-12-10T07:26:32,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592384 2024-12-10T07:26:32,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592384 2024-12-10T07:26:32,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592384) roll requested 2024-12-10T07:26:32,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592393 2024-12-10T07:26:32,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592393 2024-12-10T07:26:32,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592393) roll requested 2024-12-10T07:26:32,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592403 2024-12-10T07:26:32,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592403 2024-12-10T07:26:32,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592403) roll requested 2024-12-10T07:26:32,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592413 2024-12-10T07:26:32,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592413 2024-12-10T07:26:32,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592413) roll requested 2024-12-10T07:26:32,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592421 2024-12-10T07:26:32,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592421 2024-12-10T07:26:32,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592421) roll requested 2024-12-10T07:26:32,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592426 2024-12-10T07:26:32,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592426 2024-12-10T07:26:32,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592426) roll requested 2024-12-10T07:26:32,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592433 2024-12-10T07:26:32,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592433 2024-12-10T07:26:32,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592433) roll requested 2024-12-10T07:26:32,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592438 2024-12-10T07:26:32,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592438 2024-12-10T07:26:32,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592438) roll requested 2024-12-10T07:26:32,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592443 2024-12-10T07:26:32,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592443 2024-12-10T07:26:32,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592443) roll requested 2024-12-10T07:26:32,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592448 2024-12-10T07:26:32,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592448 2024-12-10T07:26:32,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592448) roll requested 2024-12-10T07:26:32,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592454 2024-12-10T07:26:32,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592454 2024-12-10T07:26:32,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592454) roll requested 2024-12-10T07:26:32,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592460 2024-12-10T07:26:32,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592460 2024-12-10T07:26:32,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592460) roll requested 2024-12-10T07:26:32,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592467 2024-12-10T07:26:32,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592467 2024-12-10T07:26:32,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592467) roll requested 2024-12-10T07:26:32,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592475 2024-12-10T07:26:32,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592475 2024-12-10T07:26:32,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592475) roll requested 2024-12-10T07:26:32,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592485 2024-12-10T07:26:32,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592485 2024-12-10T07:26:32,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592485) roll requested 2024-12-10T07:26:32,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592492 2024-12-10T07:26:32,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592492 2024-12-10T07:26:32,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592492) roll requested 2024-12-10T07:26:32,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592501 2024-12-10T07:26:32,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592501 2024-12-10T07:26:32,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592501) roll requested 2024-12-10T07:26:32,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592507 2024-12-10T07:26:32,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592507 2024-12-10T07:26:32,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592507) roll requested 2024-12-10T07:26:32,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592514 2024-12-10T07:26:32,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592514 2024-12-10T07:26:32,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592514) roll requested 2024-12-10T07:26:32,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592522 2024-12-10T07:26:32,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592522 2024-12-10T07:26:32,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592522) roll requested 2024-12-10T07:26:32,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592530 2024-12-10T07:26:32,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592530 2024-12-10T07:26:32,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592530) roll requested 2024-12-10T07:26:32,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592539 2024-12-10T07:26:32,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592539 2024-12-10T07:26:32,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592539) roll requested 2024-12-10T07:26:32,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592548 2024-12-10T07:26:32,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592548 2024-12-10T07:26:32,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592548) roll requested 2024-12-10T07:26:32,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592557 2024-12-10T07:26:32,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592557 2024-12-10T07:26:32,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592557) roll requested 2024-12-10T07:26:32,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592564 2024-12-10T07:26:32,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592564 2024-12-10T07:26:32,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592564) roll requested 2024-12-10T07:26:32,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592570 2024-12-10T07:26:32,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592570 2024-12-10T07:26:32,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592570) roll requested 2024-12-10T07:26:32,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592579 2024-12-10T07:26:32,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592579 2024-12-10T07:26:32,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592579) roll requested 2024-12-10T07:26:32,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592585 2024-12-10T07:26:32,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592585 2024-12-10T07:26:32,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592585) roll requested 2024-12-10T07:26:32,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592592 2024-12-10T07:26:32,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592592 2024-12-10T07:26:32,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592592) roll requested 2024-12-10T07:26:32,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592600 2024-12-10T07:26:32,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592600 2024-12-10T07:26:32,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592600) roll requested 2024-12-10T07:26:32,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592609 2024-12-10T07:26:32,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592609 2024-12-10T07:26:32,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592609) roll requested 2024-12-10T07:26:32,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592616 2024-12-10T07:26:32,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592616 2024-12-10T07:26:32,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592616) roll requested 2024-12-10T07:26:32,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592623 2024-12-10T07:26:32,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592623 2024-12-10T07:26:32,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592623) roll requested 2024-12-10T07:26:32,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592628 2024-12-10T07:26:32,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592628 2024-12-10T07:26:32,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592628) roll requested 2024-12-10T07:26:32,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592634 2024-12-10T07:26:32,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592634 2024-12-10T07:26:32,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592634) roll requested 2024-12-10T07:26:32,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592642 2024-12-10T07:26:32,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592642 2024-12-10T07:26:32,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592642) roll requested 2024-12-10T07:26:32,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592650 2024-12-10T07:26:32,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592650 2024-12-10T07:26:32,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592650) roll requested 2024-12-10T07:26:32,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592658 2024-12-10T07:26:32,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592658 2024-12-10T07:26:32,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592658) roll requested 2024-12-10T07:26:32,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592664 2024-12-10T07:26:32,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592664 2024-12-10T07:26:32,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592664) roll requested 2024-12-10T07:26:32,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592670 2024-12-10T07:26:32,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592670 2024-12-10T07:26:32,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592670) roll requested 2024-12-10T07:26:32,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592676 2024-12-10T07:26:32,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592676 2024-12-10T07:26:32,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592676) roll requested 2024-12-10T07:26:32,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592683 2024-12-10T07:26:32,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592683 2024-12-10T07:26:32,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592683) roll requested 2024-12-10T07:26:32,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592697 2024-12-10T07:26:32,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592697 2024-12-10T07:26:32,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592697) roll requested 2024-12-10T07:26:32,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592704 2024-12-10T07:26:32,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592704 2024-12-10T07:26:32,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592704) roll requested 2024-12-10T07:26:32,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592714 2024-12-10T07:26:32,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592714 2024-12-10T07:26:32,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592714) roll requested 2024-12-10T07:26:32,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592721 2024-12-10T07:26:32,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592721 2024-12-10T07:26:32,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592721) roll requested 2024-12-10T07:26:32,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592729 2024-12-10T07:26:32,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592729 2024-12-10T07:26:32,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592729) roll requested 2024-12-10T07:26:32,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592737 2024-12-10T07:26:32,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592737 2024-12-10T07:26:32,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592737) roll requested 2024-12-10T07:26:32,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592745 2024-12-10T07:26:32,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592745 2024-12-10T07:26:32,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592745) roll requested 2024-12-10T07:26:32,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592754 2024-12-10T07:26:32,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592754 2024-12-10T07:26:32,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592754) roll requested 2024-12-10T07:26:32,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592759 2024-12-10T07:26:32,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592759 2024-12-10T07:26:32,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592759) roll requested 2024-12-10T07:26:32,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592764 2024-12-10T07:26:32,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592764 2024-12-10T07:26:32,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592764) roll requested 2024-12-10T07:26:32,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592769 2024-12-10T07:26:32,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592769 2024-12-10T07:26:32,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592769) roll requested 2024-12-10T07:26:32,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592774 2024-12-10T07:26:32,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592774 2024-12-10T07:26:32,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592774) roll requested 2024-12-10T07:26:32,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592779 2024-12-10T07:26:32,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592779 2024-12-10T07:26:32,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592779) roll requested 2024-12-10T07:26:32,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592784 2024-12-10T07:26:32,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592784 2024-12-10T07:26:32,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592784) roll requested 2024-12-10T07:26:32,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592788 2024-12-10T07:26:32,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592788 2024-12-10T07:26:32,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592788) roll requested 2024-12-10T07:26:32,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592793 2024-12-10T07:26:32,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592793 2024-12-10T07:26:32,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592793) roll requested 2024-12-10T07:26:32,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592800 2024-12-10T07:26:32,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592800 2024-12-10T07:26:32,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592800) roll requested 2024-12-10T07:26:32,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592806 2024-12-10T07:26:32,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592806 2024-12-10T07:26:32,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592806) roll requested 2024-12-10T07:26:32,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592814 2024-12-10T07:26:32,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592814 2024-12-10T07:26:32,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592814) roll requested 2024-12-10T07:26:32,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592822 2024-12-10T07:26:32,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592822 2024-12-10T07:26:32,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592822) roll requested 2024-12-10T07:26:32,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592830 2024-12-10T07:26:32,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592830 2024-12-10T07:26:32,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592830) roll requested 2024-12-10T07:26:32,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592839 2024-12-10T07:26:32,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592839 2024-12-10T07:26:32,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592839) roll requested 2024-12-10T07:26:32,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592847 2024-12-10T07:26:32,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592847 2024-12-10T07:26:32,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592847) roll requested 2024-12-10T07:26:32,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592855 2024-12-10T07:26:32,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592855 2024-12-10T07:26:32,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592855) roll requested 2024-12-10T07:26:32,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592864 2024-12-10T07:26:32,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592864 2024-12-10T07:26:32,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592864) roll requested 2024-12-10T07:26:32,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592874 2024-12-10T07:26:32,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592874 2024-12-10T07:26:32,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592874) roll requested 2024-12-10T07:26:32,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592884 2024-12-10T07:26:32,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592884 2024-12-10T07:26:32,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592884) roll requested 2024-12-10T07:26:32,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592893 2024-12-10T07:26:32,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592893 2024-12-10T07:26:32,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592893) roll requested 2024-12-10T07:26:32,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592903 2024-12-10T07:26:32,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592903 2024-12-10T07:26:32,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592903) roll requested 2024-12-10T07:26:32,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592912 2024-12-10T07:26:32,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592912 2024-12-10T07:26:32,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592912) roll requested 2024-12-10T07:26:32,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592921 2024-12-10T07:26:32,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592921 2024-12-10T07:26:32,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592921) roll requested 2024-12-10T07:26:32,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592929 2024-12-10T07:26:32,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592929 2024-12-10T07:26:32,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592929) roll requested 2024-12-10T07:26:32,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592937 2024-12-10T07:26:32,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592937 2024-12-10T07:26:32,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592937) roll requested 2024-12-10T07:26:32,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592944 2024-12-10T07:26:32,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592944 2024-12-10T07:26:32,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592944) roll requested 2024-12-10T07:26:32,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592950 2024-12-10T07:26:32,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592950 2024-12-10T07:26:32,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592950) roll requested 2024-12-10T07:26:32,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592959 2024-12-10T07:26:32,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592959 2024-12-10T07:26:32,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592959) roll requested 2024-12-10T07:26:32,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592967 2024-12-10T07:26:32,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592967 2024-12-10T07:26:32,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592967) roll requested 2024-12-10T07:26:32,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592977 2024-12-10T07:26:32,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592977 2024-12-10T07:26:32,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592977) roll requested 2024-12-10T07:26:32,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592987 2024-12-10T07:26:32,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592987 2024-12-10T07:26:32,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592987) roll requested 2024-12-10T07:26:32,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592992 2024-12-10T07:26:32,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:32,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592992 2024-12-10T07:26:32,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:32,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:32,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592992) roll requested 2024-12-10T07:26:32,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815592999 2024-12-10T07:26:33,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592999 2024-12-10T07:26:33,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815592999) roll requested 2024-12-10T07:26:33,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593006 2024-12-10T07:26:33,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815592999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593006 2024-12-10T07:26:33,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593006) roll requested 2024-12-10T07:26:33,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593014 2024-12-10T07:26:33,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593014 2024-12-10T07:26:33,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593014) roll requested 2024-12-10T07:26:33,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593021 2024-12-10T07:26:33,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593021 2024-12-10T07:26:33,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593021) roll requested 2024-12-10T07:26:33,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593029 2024-12-10T07:26:33,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593029 2024-12-10T07:26:33,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593029) roll requested 2024-12-10T07:26:33,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593035 2024-12-10T07:26:33,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593035 2024-12-10T07:26:33,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593035) roll requested 2024-12-10T07:26:33,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593044 2024-12-10T07:26:33,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593044 2024-12-10T07:26:33,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593044) roll requested 2024-12-10T07:26:33,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593052 2024-12-10T07:26:33,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593052 2024-12-10T07:26:33,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593052) roll requested 2024-12-10T07:26:33,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593060 2024-12-10T07:26:33,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593060 2024-12-10T07:26:33,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593060) roll requested 2024-12-10T07:26:33,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593070 2024-12-10T07:26:33,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593070 2024-12-10T07:26:33,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593070) roll requested 2024-12-10T07:26:33,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593076 2024-12-10T07:26:33,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593076 2024-12-10T07:26:33,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593076) roll requested 2024-12-10T07:26:33,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593081 2024-12-10T07:26:33,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593081 2024-12-10T07:26:33,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593081) roll requested 2024-12-10T07:26:33,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593085 2024-12-10T07:26:33,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593085 2024-12-10T07:26:33,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593085) roll requested 2024-12-10T07:26:33,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593090 2024-12-10T07:26:33,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593090 2024-12-10T07:26:33,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593090) roll requested 2024-12-10T07:26:33,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593096 2024-12-10T07:26:33,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593096 2024-12-10T07:26:33,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593096) roll requested 2024-12-10T07:26:33,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593103 2024-12-10T07:26:33,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593103 2024-12-10T07:26:33,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593103) roll requested 2024-12-10T07:26:33,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593110 2024-12-10T07:26:33,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593110 2024-12-10T07:26:33,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593110) roll requested 2024-12-10T07:26:33,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593119 2024-12-10T07:26:33,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593119 2024-12-10T07:26:33,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593119) roll requested 2024-12-10T07:26:33,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593127 2024-12-10T07:26:33,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593127 2024-12-10T07:26:33,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593127) roll requested 2024-12-10T07:26:33,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593133 2024-12-10T07:26:33,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593133 2024-12-10T07:26:33,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593133) roll requested 2024-12-10T07:26:33,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593140 2024-12-10T07:26:33,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593140 2024-12-10T07:26:33,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593140) roll requested 2024-12-10T07:26:33,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593148 2024-12-10T07:26:33,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593148 2024-12-10T07:26:33,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593148) roll requested 2024-12-10T07:26:33,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593157 2024-12-10T07:26:33,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593157 2024-12-10T07:26:33,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593157) roll requested 2024-12-10T07:26:33,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593165 2024-12-10T07:26:33,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593165 2024-12-10T07:26:33,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593165) roll requested 2024-12-10T07:26:33,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593172 2024-12-10T07:26:33,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593172 2024-12-10T07:26:33,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593172) roll requested 2024-12-10T07:26:33,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593179 2024-12-10T07:26:33,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593179 2024-12-10T07:26:33,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593179) roll requested 2024-12-10T07:26:33,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593188 2024-12-10T07:26:33,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593188 2024-12-10T07:26:33,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593188) roll requested 2024-12-10T07:26:33,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593196 2024-12-10T07:26:33,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593196 2024-12-10T07:26:33,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593196) roll requested 2024-12-10T07:26:33,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593206 2024-12-10T07:26:33,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593206 2024-12-10T07:26:33,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593206) roll requested 2024-12-10T07:26:33,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593213 2024-12-10T07:26:33,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593213 2024-12-10T07:26:33,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593213) roll requested 2024-12-10T07:26:33,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593219 2024-12-10T07:26:33,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593219 2024-12-10T07:26:33,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593219) roll requested 2024-12-10T07:26:33,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593224 2024-12-10T07:26:33,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593224 2024-12-10T07:26:33,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593224) roll requested 2024-12-10T07:26:33,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593230 2024-12-10T07:26:33,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593230 2024-12-10T07:26:33,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593230) roll requested 2024-12-10T07:26:33,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593236 2024-12-10T07:26:33,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593236 2024-12-10T07:26:33,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593236) roll requested 2024-12-10T07:26:33,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593241 2024-12-10T07:26:33,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593241 2024-12-10T07:26:33,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593241) roll requested 2024-12-10T07:26:33,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593247 2024-12-10T07:26:33,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593247 2024-12-10T07:26:33,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593247) roll requested 2024-12-10T07:26:33,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593254 2024-12-10T07:26:33,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593254 2024-12-10T07:26:33,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593254) roll requested 2024-12-10T07:26:33,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593260 2024-12-10T07:26:33,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593260 2024-12-10T07:26:33,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593260) roll requested 2024-12-10T07:26:33,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593266 2024-12-10T07:26:33,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593266 2024-12-10T07:26:33,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593266) roll requested 2024-12-10T07:26:33,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593272 2024-12-10T07:26:33,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593272 2024-12-10T07:26:33,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593272) roll requested 2024-12-10T07:26:33,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593277 2024-12-10T07:26:33,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593277 2024-12-10T07:26:33,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593277) roll requested 2024-12-10T07:26:33,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593283 2024-12-10T07:26:33,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593283 2024-12-10T07:26:33,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593283) roll requested 2024-12-10T07:26:33,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593289 2024-12-10T07:26:33,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593289 2024-12-10T07:26:33,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593289) roll requested 2024-12-10T07:26:33,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593295 2024-12-10T07:26:33,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593295 2024-12-10T07:26:33,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593295) roll requested 2024-12-10T07:26:33,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593301 2024-12-10T07:26:33,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593301 2024-12-10T07:26:33,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593301) roll requested 2024-12-10T07:26:33,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593307 2024-12-10T07:26:33,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593307 2024-12-10T07:26:33,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593307) roll requested 2024-12-10T07:26:33,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593313 2024-12-10T07:26:33,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593313 2024-12-10T07:26:33,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593313) roll requested 2024-12-10T07:26:33,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593318 2024-12-10T07:26:33,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593318 2024-12-10T07:26:33,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593318) roll requested 2024-12-10T07:26:33,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593324 2024-12-10T07:26:33,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593324 2024-12-10T07:26:33,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593324) roll requested 2024-12-10T07:26:33,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593330 2024-12-10T07:26:33,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593330 2024-12-10T07:26:33,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593330) roll requested 2024-12-10T07:26:33,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593336 2024-12-10T07:26:33,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593336 2024-12-10T07:26:33,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593336) roll requested 2024-12-10T07:26:33,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593341 2024-12-10T07:26:33,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593341 2024-12-10T07:26:33,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593341) roll requested 2024-12-10T07:26:33,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593346 2024-12-10T07:26:33,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593346 2024-12-10T07:26:33,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593346) roll requested 2024-12-10T07:26:33,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593351 2024-12-10T07:26:33,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593351 2024-12-10T07:26:33,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593351) roll requested 2024-12-10T07:26:33,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593357 2024-12-10T07:26:33,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593357 2024-12-10T07:26:33,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593357) roll requested 2024-12-10T07:26:33,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593364 2024-12-10T07:26:33,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593364 2024-12-10T07:26:33,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593364) roll requested 2024-12-10T07:26:33,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593372 2024-12-10T07:26:33,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593372 2024-12-10T07:26:33,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593372) roll requested 2024-12-10T07:26:33,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593385 2024-12-10T07:26:33,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593385 2024-12-10T07:26:33,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593385) roll requested 2024-12-10T07:26:33,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593390 2024-12-10T07:26:33,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593390 2024-12-10T07:26:33,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593390) roll requested 2024-12-10T07:26:33,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593395 2024-12-10T07:26:33,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593395 2024-12-10T07:26:33,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593395) roll requested 2024-12-10T07:26:33,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593400 2024-12-10T07:26:33,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593400 2024-12-10T07:26:33,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593400) roll requested 2024-12-10T07:26:33,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593406 2024-12-10T07:26:33,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593406 2024-12-10T07:26:33,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593406) roll requested 2024-12-10T07:26:33,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593412 2024-12-10T07:26:33,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593412 2024-12-10T07:26:33,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593412) roll requested 2024-12-10T07:26:33,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593420 2024-12-10T07:26:33,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593420 2024-12-10T07:26:33,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593420) roll requested 2024-12-10T07:26:33,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593428 2024-12-10T07:26:33,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593428 2024-12-10T07:26:33,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593428) roll requested 2024-12-10T07:26:33,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593437 2024-12-10T07:26:33,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593437 2024-12-10T07:26:33,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593437) roll requested 2024-12-10T07:26:33,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593447 2024-12-10T07:26:33,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593447 2024-12-10T07:26:33,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593447) roll requested 2024-12-10T07:26:33,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593454 2024-12-10T07:26:33,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593454 2024-12-10T07:26:33,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593454) roll requested 2024-12-10T07:26:33,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593460 2024-12-10T07:26:33,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593460 2024-12-10T07:26:33,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593460) roll requested 2024-12-10T07:26:33,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593467 2024-12-10T07:26:33,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593467 2024-12-10T07:26:33,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593467) roll requested 2024-12-10T07:26:33,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593478 2024-12-10T07:26:33,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593478 2024-12-10T07:26:33,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593478) roll requested 2024-12-10T07:26:33,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593484 2024-12-10T07:26:33,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593484 2024-12-10T07:26:33,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593484) roll requested 2024-12-10T07:26:33,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593490 2024-12-10T07:26:33,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593490 2024-12-10T07:26:33,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593490) roll requested 2024-12-10T07:26:33,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593497 2024-12-10T07:26:33,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593497 2024-12-10T07:26:33,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593497) roll requested 2024-12-10T07:26:33,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593505 2024-12-10T07:26:33,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593505 2024-12-10T07:26:33,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593505) roll requested 2024-12-10T07:26:33,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593513 2024-12-10T07:26:33,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593513 2024-12-10T07:26:33,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593513) roll requested 2024-12-10T07:26:33,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593522 2024-12-10T07:26:33,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593522 2024-12-10T07:26:33,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593522) roll requested 2024-12-10T07:26:33,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593529 2024-12-10T07:26:33,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593529 2024-12-10T07:26:33,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593529) roll requested 2024-12-10T07:26:33,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593537 2024-12-10T07:26:33,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593537 2024-12-10T07:26:33,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593537) roll requested 2024-12-10T07:26:33,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593545 2024-12-10T07:26:33,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593545 2024-12-10T07:26:33,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593545) roll requested 2024-12-10T07:26:33,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593554 2024-12-10T07:26:33,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593554 2024-12-10T07:26:33,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593554) roll requested 2024-12-10T07:26:33,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593561 2024-12-10T07:26:33,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593561 2024-12-10T07:26:33,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593561) roll requested 2024-12-10T07:26:33,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593568 2024-12-10T07:26:33,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593568 2024-12-10T07:26:33,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593568) roll requested 2024-12-10T07:26:33,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593574 2024-12-10T07:26:33,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593574 2024-12-10T07:26:33,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593574) roll requested 2024-12-10T07:26:33,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593581 2024-12-10T07:26:33,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593581 2024-12-10T07:26:33,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593581) roll requested 2024-12-10T07:26:33,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593589 2024-12-10T07:26:33,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593589 2024-12-10T07:26:33,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593589) roll requested 2024-12-10T07:26:33,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593599 2024-12-10T07:26:33,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593599 2024-12-10T07:26:33,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593599) roll requested 2024-12-10T07:26:33,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593609 2024-12-10T07:26:33,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593609 2024-12-10T07:26:33,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593609) roll requested 2024-12-10T07:26:33,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593617 2024-12-10T07:26:33,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593617 2024-12-10T07:26:33,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593617) roll requested 2024-12-10T07:26:33,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593625 2024-12-10T07:26:33,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593625 2024-12-10T07:26:33,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593625) roll requested 2024-12-10T07:26:33,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593634 2024-12-10T07:26:33,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593634 2024-12-10T07:26:33,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593634) roll requested 2024-12-10T07:26:33,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593641 2024-12-10T07:26:33,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593641 2024-12-10T07:26:33,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593641) roll requested 2024-12-10T07:26:33,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593648 2024-12-10T07:26:33,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593648 2024-12-10T07:26:33,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593648) roll requested 2024-12-10T07:26:33,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593655 2024-12-10T07:26:33,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593655 2024-12-10T07:26:33,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593655) roll requested 2024-12-10T07:26:33,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593661 2024-12-10T07:26:33,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593661 2024-12-10T07:26:33,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593661) roll requested 2024-12-10T07:26:33,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593668 2024-12-10T07:26:33,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593668 2024-12-10T07:26:33,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593668) roll requested 2024-12-10T07:26:33,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593676 2024-12-10T07:26:33,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593676 2024-12-10T07:26:33,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593676) roll requested 2024-12-10T07:26:33,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593685 2024-12-10T07:26:33,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593685 2024-12-10T07:26:33,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593685) roll requested 2024-12-10T07:26:33,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593692 2024-12-10T07:26:33,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593692 2024-12-10T07:26:33,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593692) roll requested 2024-12-10T07:26:33,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593699 2024-12-10T07:26:33,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593699 2024-12-10T07:26:33,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593699) roll requested 2024-12-10T07:26:33,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593706 2024-12-10T07:26:33,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593706 2024-12-10T07:26:33,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593706) roll requested 2024-12-10T07:26:33,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593714 2024-12-10T07:26:33,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593714 2024-12-10T07:26:33,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593714) roll requested 2024-12-10T07:26:33,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593723 2024-12-10T07:26:33,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593723 2024-12-10T07:26:33,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593723) roll requested 2024-12-10T07:26:33,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593729 2024-12-10T07:26:33,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593729 2024-12-10T07:26:33,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593729) roll requested 2024-12-10T07:26:33,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593735 2024-12-10T07:26:33,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593735 2024-12-10T07:26:33,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593735) roll requested 2024-12-10T07:26:33,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593743 2024-12-10T07:26:33,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593743 2024-12-10T07:26:33,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593743) roll requested 2024-12-10T07:26:33,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593751 2024-12-10T07:26:33,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593751 2024-12-10T07:26:33,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593751) roll requested 2024-12-10T07:26:33,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593759 2024-12-10T07:26:33,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593759 2024-12-10T07:26:33,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593759) roll requested 2024-12-10T07:26:33,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593768 2024-12-10T07:26:33,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593768 2024-12-10T07:26:33,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593768) roll requested 2024-12-10T07:26:33,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593777 2024-12-10T07:26:33,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593777 2024-12-10T07:26:33,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593777) roll requested 2024-12-10T07:26:33,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593782 2024-12-10T07:26:33,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593782 2024-12-10T07:26:33,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593782) roll requested 2024-12-10T07:26:33,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593786 2024-12-10T07:26:33,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593786 2024-12-10T07:26:33,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593786) roll requested 2024-12-10T07:26:33,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593792 2024-12-10T07:26:33,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593792 2024-12-10T07:26:33,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593792) roll requested 2024-12-10T07:26:33,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593800 2024-12-10T07:26:33,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593800 2024-12-10T07:26:33,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593800) roll requested 2024-12-10T07:26:33,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593807 2024-12-10T07:26:33,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593807 2024-12-10T07:26:33,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593807) roll requested 2024-12-10T07:26:33,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593815 2024-12-10T07:26:33,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593815 2024-12-10T07:26:33,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593815) roll requested 2024-12-10T07:26:33,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593822 2024-12-10T07:26:33,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593822 2024-12-10T07:26:33,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593822) roll requested 2024-12-10T07:26:33,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593830 2024-12-10T07:26:33,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593830 2024-12-10T07:26:33,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593830) roll requested 2024-12-10T07:26:33,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593836 2024-12-10T07:26:33,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593836 2024-12-10T07:26:33,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593836) roll requested 2024-12-10T07:26:33,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593841 2024-12-10T07:26:33,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593841 2024-12-10T07:26:33,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593841) roll requested 2024-12-10T07:26:33,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593849 2024-12-10T07:26:33,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593849 2024-12-10T07:26:33,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593849) roll requested 2024-12-10T07:26:33,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593857 2024-12-10T07:26:33,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593857 2024-12-10T07:26:33,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593857) roll requested 2024-12-10T07:26:33,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593864 2024-12-10T07:26:33,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593864 2024-12-10T07:26:33,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593864) roll requested 2024-12-10T07:26:33,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593872 2024-12-10T07:26:33,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593872 2024-12-10T07:26:33,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593872) roll requested 2024-12-10T07:26:33,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593880 2024-12-10T07:26:33,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593880 2024-12-10T07:26:33,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593880) roll requested 2024-12-10T07:26:33,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593887 2024-12-10T07:26:33,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593887 2024-12-10T07:26:33,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593887) roll requested 2024-12-10T07:26:33,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593895 2024-12-10T07:26:33,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593895 2024-12-10T07:26:33,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593895) roll requested 2024-12-10T07:26:33,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593904 2024-12-10T07:26:33,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593904 2024-12-10T07:26:33,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593904) roll requested 2024-12-10T07:26:33,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593911 2024-12-10T07:26:33,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593911 2024-12-10T07:26:33,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593911) roll requested 2024-12-10T07:26:33,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593919 2024-12-10T07:26:33,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593919 2024-12-10T07:26:33,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593919) roll requested 2024-12-10T07:26:33,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593927 2024-12-10T07:26:33,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593927 2024-12-10T07:26:33,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593927) roll requested 2024-12-10T07:26:33,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593936 2024-12-10T07:26:33,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593936 2024-12-10T07:26:33,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593936) roll requested 2024-12-10T07:26:33,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593945 2024-12-10T07:26:33,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593945 2024-12-10T07:26:33,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593945) roll requested 2024-12-10T07:26:33,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593952 2024-12-10T07:26:33,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593952 2024-12-10T07:26:33,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593952) roll requested 2024-12-10T07:26:33,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593960 2024-12-10T07:26:33,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593960 2024-12-10T07:26:33,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593960) roll requested 2024-12-10T07:26:33,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593968 2024-12-10T07:26:33,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593968 2024-12-10T07:26:33,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593968) roll requested 2024-12-10T07:26:33,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593975 2024-12-10T07:26:33,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593975 2024-12-10T07:26:33,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593975) roll requested 2024-12-10T07:26:33,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593983 2024-12-10T07:26:33,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593983 2024-12-10T07:26:33,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:33,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:33,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593983) roll requested 2024-12-10T07:26:33,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815593992 2024-12-10T07:26:33,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:33,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593992 2024-12-10T07:26:34,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815593992) roll requested 2024-12-10T07:26:34,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594001 2024-12-10T07:26:34,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815593992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594001 2024-12-10T07:26:34,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594001) roll requested 2024-12-10T07:26:34,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594010 2024-12-10T07:26:34,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594010 2024-12-10T07:26:34,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594010) roll requested 2024-12-10T07:26:34,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594017 2024-12-10T07:26:34,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594017 2024-12-10T07:26:34,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594017) roll requested 2024-12-10T07:26:34,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594025 2024-12-10T07:26:34,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594025 2024-12-10T07:26:34,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594025) roll requested 2024-12-10T07:26:34,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594034 2024-12-10T07:26:34,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594034 2024-12-10T07:26:34,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594034) roll requested 2024-12-10T07:26:34,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594043 2024-12-10T07:26:34,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594043 2024-12-10T07:26:34,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594043) roll requested 2024-12-10T07:26:34,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594049 2024-12-10T07:26:34,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594049 2024-12-10T07:26:34,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594049) roll requested 2024-12-10T07:26:34,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594056 2024-12-10T07:26:34,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594056 2024-12-10T07:26:34,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594056) roll requested 2024-12-10T07:26:34,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594063 2024-12-10T07:26:34,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594063 2024-12-10T07:26:34,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594063) roll requested 2024-12-10T07:26:34,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594071 2024-12-10T07:26:34,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594071 2024-12-10T07:26:34,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594071) roll requested 2024-12-10T07:26:34,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594080 2024-12-10T07:26:34,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594080 2024-12-10T07:26:34,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594080) roll requested 2024-12-10T07:26:34,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594090 2024-12-10T07:26:34,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594090 2024-12-10T07:26:34,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594090) roll requested 2024-12-10T07:26:34,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594098 2024-12-10T07:26:34,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594098 2024-12-10T07:26:34,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594098) roll requested 2024-12-10T07:26:34,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594105 2024-12-10T07:26:34,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594105 2024-12-10T07:26:34,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594105) roll requested 2024-12-10T07:26:34,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594113 2024-12-10T07:26:34,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594113 2024-12-10T07:26:34,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594113) roll requested 2024-12-10T07:26:34,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594121 2024-12-10T07:26:34,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594121 2024-12-10T07:26:34,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594121) roll requested 2024-12-10T07:26:34,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594131 2024-12-10T07:26:34,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594131 2024-12-10T07:26:34,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594131) roll requested 2024-12-10T07:26:34,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594141 2024-12-10T07:26:34,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594141 2024-12-10T07:26:34,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594141) roll requested 2024-12-10T07:26:34,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594150 2024-12-10T07:26:34,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594150 2024-12-10T07:26:34,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594150) roll requested 2024-12-10T07:26:34,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594157 2024-12-10T07:26:34,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594157 2024-12-10T07:26:34,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594157) roll requested 2024-12-10T07:26:34,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594165 2024-12-10T07:26:34,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594165 2024-12-10T07:26:34,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594165) roll requested 2024-12-10T07:26:34,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594172 2024-12-10T07:26:34,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594172 2024-12-10T07:26:34,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594172) roll requested 2024-12-10T07:26:34,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594180 2024-12-10T07:26:34,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594180 2024-12-10T07:26:34,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594180) roll requested 2024-12-10T07:26:34,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594189 2024-12-10T07:26:34,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594189 2024-12-10T07:26:34,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594189) roll requested 2024-12-10T07:26:34,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594195 2024-12-10T07:26:34,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594195 2024-12-10T07:26:34,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594195) roll requested 2024-12-10T07:26:34,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594201 2024-12-10T07:26:34,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594201 2024-12-10T07:26:34,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594201) roll requested 2024-12-10T07:26:34,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594210 2024-12-10T07:26:34,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594210 2024-12-10T07:26:34,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594210) roll requested 2024-12-10T07:26:34,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594218 2024-12-10T07:26:34,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594218 2024-12-10T07:26:34,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594218) roll requested 2024-12-10T07:26:34,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594225 2024-12-10T07:26:34,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594225 2024-12-10T07:26:34,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594225) roll requested 2024-12-10T07:26:34,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594232 2024-12-10T07:26:34,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594232 2024-12-10T07:26:34,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594232) roll requested 2024-12-10T07:26:34,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594240 2024-12-10T07:26:34,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594240 2024-12-10T07:26:34,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594240) roll requested 2024-12-10T07:26:34,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594249 2024-12-10T07:26:34,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594249 2024-12-10T07:26:34,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594249) roll requested 2024-12-10T07:26:34,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594254 2024-12-10T07:26:34,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594254 2024-12-10T07:26:34,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594254) roll requested 2024-12-10T07:26:34,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594258 2024-12-10T07:26:34,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594258 2024-12-10T07:26:34,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594258) roll requested 2024-12-10T07:26:34,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594265 2024-12-10T07:26:34,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594265 2024-12-10T07:26:34,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594265) roll requested 2024-12-10T07:26:34,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594271 2024-12-10T07:26:34,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594271 2024-12-10T07:26:34,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594271) roll requested 2024-12-10T07:26:34,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594278 2024-12-10T07:26:34,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594278 2024-12-10T07:26:34,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594278) roll requested 2024-12-10T07:26:34,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594286 2024-12-10T07:26:34,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594286 2024-12-10T07:26:34,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594286) roll requested 2024-12-10T07:26:34,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594294 2024-12-10T07:26:34,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594294 2024-12-10T07:26:34,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594294) roll requested 2024-12-10T07:26:34,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594300 2024-12-10T07:26:34,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594300 2024-12-10T07:26:34,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594300) roll requested 2024-12-10T07:26:34,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594306 2024-12-10T07:26:34,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594306 2024-12-10T07:26:34,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594306) roll requested 2024-12-10T07:26:34,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594311 2024-12-10T07:26:34,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594311 2024-12-10T07:26:34,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594311) roll requested 2024-12-10T07:26:34,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594319 2024-12-10T07:26:34,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594319 2024-12-10T07:26:34,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594319) roll requested 2024-12-10T07:26:34,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594326 2024-12-10T07:26:34,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594326 2024-12-10T07:26:34,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594326) roll requested 2024-12-10T07:26:34,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594332 2024-12-10T07:26:34,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594332 2024-12-10T07:26:34,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594332) roll requested 2024-12-10T07:26:34,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594340 2024-12-10T07:26:34,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594340 2024-12-10T07:26:34,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594340) roll requested 2024-12-10T07:26:34,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594347 2024-12-10T07:26:34,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594347 2024-12-10T07:26:34,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594347) roll requested 2024-12-10T07:26:34,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594354 2024-12-10T07:26:34,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594354 2024-12-10T07:26:34,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594354) roll requested 2024-12-10T07:26:34,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594362 2024-12-10T07:26:34,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594362 2024-12-10T07:26:34,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594362) roll requested 2024-12-10T07:26:34,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594371 2024-12-10T07:26:34,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594371 2024-12-10T07:26:34,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594371) roll requested 2024-12-10T07:26:34,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594380 2024-12-10T07:26:34,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594380 2024-12-10T07:26:34,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594380) roll requested 2024-12-10T07:26:34,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594389 2024-12-10T07:26:34,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594389 2024-12-10T07:26:34,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594389) roll requested 2024-12-10T07:26:34,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594399 2024-12-10T07:26:34,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594399 2024-12-10T07:26:34,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594399) roll requested 2024-12-10T07:26:34,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594408 2024-12-10T07:26:34,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594408 2024-12-10T07:26:34,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594408) roll requested 2024-12-10T07:26:34,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594416 2024-12-10T07:26:34,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594416 2024-12-10T07:26:34,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594416) roll requested 2024-12-10T07:26:34,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594424 2024-12-10T07:26:34,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594424 2024-12-10T07:26:34,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594424) roll requested 2024-12-10T07:26:34,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594433 2024-12-10T07:26:34,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594433 2024-12-10T07:26:34,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594433) roll requested 2024-12-10T07:26:34,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594443 2024-12-10T07:26:34,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594443 2024-12-10T07:26:34,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594443) roll requested 2024-12-10T07:26:34,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594450 2024-12-10T07:26:34,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594450 2024-12-10T07:26:34,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594450) roll requested 2024-12-10T07:26:34,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594457 2024-12-10T07:26:34,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594457 2024-12-10T07:26:34,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594457) roll requested 2024-12-10T07:26:34,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594466 2024-12-10T07:26:34,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594466 2024-12-10T07:26:34,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594466) roll requested 2024-12-10T07:26:34,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594474 2024-12-10T07:26:34,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594474 2024-12-10T07:26:34,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594474) roll requested 2024-12-10T07:26:34,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594483 2024-12-10T07:26:34,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594483 2024-12-10T07:26:34,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594483) roll requested 2024-12-10T07:26:34,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594493 2024-12-10T07:26:34,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594493 2024-12-10T07:26:34,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594493) roll requested 2024-12-10T07:26:34,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594501 2024-12-10T07:26:34,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594501 2024-12-10T07:26:34,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594501) roll requested 2024-12-10T07:26:34,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594508 2024-12-10T07:26:34,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594508 2024-12-10T07:26:34,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594508) roll requested 2024-12-10T07:26:34,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594514 2024-12-10T07:26:34,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594514 2024-12-10T07:26:34,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594514) roll requested 2024-12-10T07:26:34,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594522 2024-12-10T07:26:34,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594522 2024-12-10T07:26:34,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594522) roll requested 2024-12-10T07:26:34,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594527 2024-12-10T07:26:34,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594527 2024-12-10T07:26:34,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594527) roll requested 2024-12-10T07:26:34,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594532 2024-12-10T07:26:34,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594532 2024-12-10T07:26:34,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594532) roll requested 2024-12-10T07:26:34,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594537 2024-12-10T07:26:34,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594537 2024-12-10T07:26:34,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594537) roll requested 2024-12-10T07:26:34,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594544 2024-12-10T07:26:34,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594544 2024-12-10T07:26:34,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594544) roll requested 2024-12-10T07:26:34,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594550 2024-12-10T07:26:34,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594550 2024-12-10T07:26:34,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594550) roll requested 2024-12-10T07:26:34,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594559 2024-12-10T07:26:34,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594559 2024-12-10T07:26:34,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594559) roll requested 2024-12-10T07:26:34,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594567 2024-12-10T07:26:34,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594567 2024-12-10T07:26:34,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594567) roll requested 2024-12-10T07:26:34,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594576 2024-12-10T07:26:34,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594576 2024-12-10T07:26:34,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594576) roll requested 2024-12-10T07:26:34,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594582 2024-12-10T07:26:34,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594582 2024-12-10T07:26:34,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594582) roll requested 2024-12-10T07:26:34,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594589 2024-12-10T07:26:34,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594589 2024-12-10T07:26:34,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594589) roll requested 2024-12-10T07:26:34,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594595 2024-12-10T07:26:34,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594595 2024-12-10T07:26:34,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594595) roll requested 2024-12-10T07:26:34,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594601 2024-12-10T07:26:34,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594601 2024-12-10T07:26:34,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594601) roll requested 2024-12-10T07:26:34,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594607 2024-12-10T07:26:34,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594607 2024-12-10T07:26:34,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594607) roll requested 2024-12-10T07:26:34,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594616 2024-12-10T07:26:34,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594616 2024-12-10T07:26:34,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594616) roll requested 2024-12-10T07:26:34,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594622 2024-12-10T07:26:34,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594622 2024-12-10T07:26:34,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594622) roll requested 2024-12-10T07:26:34,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594627 2024-12-10T07:26:34,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594627 2024-12-10T07:26:34,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594627) roll requested 2024-12-10T07:26:34,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594634 2024-12-10T07:26:34,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594634 2024-12-10T07:26:34,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594634) roll requested 2024-12-10T07:26:34,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594641 2024-12-10T07:26:34,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594641 2024-12-10T07:26:34,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594641) roll requested 2024-12-10T07:26:34,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594647 2024-12-10T07:26:34,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594647 2024-12-10T07:26:34,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594647) roll requested 2024-12-10T07:26:34,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594652 2024-12-10T07:26:34,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594652 2024-12-10T07:26:34,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594652) roll requested 2024-12-10T07:26:34,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594657 2024-12-10T07:26:34,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594657 2024-12-10T07:26:34,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594657) roll requested 2024-12-10T07:26:34,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594662 2024-12-10T07:26:34,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594662 2024-12-10T07:26:34,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594662) roll requested 2024-12-10T07:26:34,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594667 2024-12-10T07:26:34,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594667 2024-12-10T07:26:34,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594667) roll requested 2024-12-10T07:26:34,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594673 2024-12-10T07:26:34,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594673 2024-12-10T07:26:34,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594673) roll requested 2024-12-10T07:26:34,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594678 2024-12-10T07:26:34,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594678 2024-12-10T07:26:34,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594678) roll requested 2024-12-10T07:26:34,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594683 2024-12-10T07:26:34,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594683 2024-12-10T07:26:34,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594683) roll requested 2024-12-10T07:26:34,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594688 2024-12-10T07:26:34,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594688 2024-12-10T07:26:34,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594688) roll requested 2024-12-10T07:26:34,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594693 2024-12-10T07:26:34,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594693 2024-12-10T07:26:34,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594693) roll requested 2024-12-10T07:26:34,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594701 2024-12-10T07:26:34,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594701 2024-12-10T07:26:34,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594701) roll requested 2024-12-10T07:26:34,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594709 2024-12-10T07:26:34,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594709 2024-12-10T07:26:34,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594709) roll requested 2024-12-10T07:26:34,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594715 2024-12-10T07:26:34,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594715 2024-12-10T07:26:34,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594715) roll requested 2024-12-10T07:26:34,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594721 2024-12-10T07:26:34,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594721 2024-12-10T07:26:34,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594721) roll requested 2024-12-10T07:26:34,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594727 2024-12-10T07:26:34,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594727 2024-12-10T07:26:34,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594727) roll requested 2024-12-10T07:26:34,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594734 2024-12-10T07:26:34,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594734 2024-12-10T07:26:34,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594734) roll requested 2024-12-10T07:26:34,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594740 2024-12-10T07:26:34,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594740 2024-12-10T07:26:34,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594740) roll requested 2024-12-10T07:26:34,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594746 2024-12-10T07:26:34,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594746 2024-12-10T07:26:34,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594746) roll requested 2024-12-10T07:26:34,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594752 2024-12-10T07:26:34,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594752 2024-12-10T07:26:34,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594752) roll requested 2024-12-10T07:26:34,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594758 2024-12-10T07:26:34,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594758 2024-12-10T07:26:34,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594758) roll requested 2024-12-10T07:26:34,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594764 2024-12-10T07:26:34,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594764 2024-12-10T07:26:34,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594764) roll requested 2024-12-10T07:26:34,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594770 2024-12-10T07:26:34,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594770 2024-12-10T07:26:34,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594770) roll requested 2024-12-10T07:26:34,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594776 2024-12-10T07:26:34,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594776 2024-12-10T07:26:34,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594776) roll requested 2024-12-10T07:26:34,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594782 2024-12-10T07:26:34,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594782 2024-12-10T07:26:34,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594782) roll requested 2024-12-10T07:26:34,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594788 2024-12-10T07:26:34,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594788 2024-12-10T07:26:34,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594788) roll requested 2024-12-10T07:26:34,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594795 2024-12-10T07:26:34,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594795 2024-12-10T07:26:34,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594795) roll requested 2024-12-10T07:26:34,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594801 2024-12-10T07:26:34,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594801 2024-12-10T07:26:34,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594801) roll requested 2024-12-10T07:26:34,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594808 2024-12-10T07:26:34,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594808 2024-12-10T07:26:34,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594808) roll requested 2024-12-10T07:26:34,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594815 2024-12-10T07:26:34,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594815 2024-12-10T07:26:34,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594815) roll requested 2024-12-10T07:26:34,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594821 2024-12-10T07:26:34,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594821 2024-12-10T07:26:34,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594821) roll requested 2024-12-10T07:26:34,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594827 2024-12-10T07:26:34,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594827 2024-12-10T07:26:34,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594827) roll requested 2024-12-10T07:26:34,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594833 2024-12-10T07:26:34,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594833 2024-12-10T07:26:34,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594833) roll requested 2024-12-10T07:26:34,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594838 2024-12-10T07:26:34,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594838 2024-12-10T07:26:34,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594838) roll requested 2024-12-10T07:26:34,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594845 2024-12-10T07:26:34,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594845 2024-12-10T07:26:34,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594845) roll requested 2024-12-10T07:26:34,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594851 2024-12-10T07:26:34,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594851 2024-12-10T07:26:34,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594851) roll requested 2024-12-10T07:26:34,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594860 2024-12-10T07:26:34,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594860 2024-12-10T07:26:34,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594860) roll requested 2024-12-10T07:26:34,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594866 2024-12-10T07:26:34,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594866 2024-12-10T07:26:34,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594866) roll requested 2024-12-10T07:26:34,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594872 2024-12-10T07:26:34,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594872 2024-12-10T07:26:34,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594872) roll requested 2024-12-10T07:26:34,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594878 2024-12-10T07:26:34,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594878 2024-12-10T07:26:34,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594878) roll requested 2024-12-10T07:26:34,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594884 2024-12-10T07:26:34,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594884 2024-12-10T07:26:34,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594884) roll requested 2024-12-10T07:26:34,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594890 2024-12-10T07:26:34,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594890 2024-12-10T07:26:34,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594890) roll requested 2024-12-10T07:26:34,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594896 2024-12-10T07:26:34,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594896 2024-12-10T07:26:34,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594896) roll requested 2024-12-10T07:26:34,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594902 2024-12-10T07:26:34,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594902 2024-12-10T07:26:34,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594902) roll requested 2024-12-10T07:26:34,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594908 2024-12-10T07:26:34,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594908 2024-12-10T07:26:34,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594908) roll requested 2024-12-10T07:26:34,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594916 2024-12-10T07:26:34,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594916 2024-12-10T07:26:34,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594916) roll requested 2024-12-10T07:26:34,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594921 2024-12-10T07:26:34,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594921 2024-12-10T07:26:34,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594921) roll requested 2024-12-10T07:26:34,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594927 2024-12-10T07:26:34,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594927 2024-12-10T07:26:34,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594927) roll requested 2024-12-10T07:26:34,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594934 2024-12-10T07:26:34,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594934 2024-12-10T07:26:34,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594934) roll requested 2024-12-10T07:26:34,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594943 2024-12-10T07:26:34,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594943 2024-12-10T07:26:34,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594943) roll requested 2024-12-10T07:26:34,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594948 2024-12-10T07:26:34,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594948 2024-12-10T07:26:34,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594948) roll requested 2024-12-10T07:26:34,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594954 2024-12-10T07:26:34,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594954 2024-12-10T07:26:34,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594954) roll requested 2024-12-10T07:26:34,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594963 2024-12-10T07:26:34,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594963 2024-12-10T07:26:34,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594963) roll requested 2024-12-10T07:26:34,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594970 2024-12-10T07:26:34,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594970 2024-12-10T07:26:34,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594970) roll requested 2024-12-10T07:26:34,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594975 2024-12-10T07:26:34,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594975 2024-12-10T07:26:34,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594975) roll requested 2024-12-10T07:26:34,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594981 2024-12-10T07:26:34,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594981 2024-12-10T07:26:34,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594981) roll requested 2024-12-10T07:26:34,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594987 2024-12-10T07:26:34,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594987 2024-12-10T07:26:34,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594987) roll requested 2024-12-10T07:26:34,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594992 2024-12-10T07:26:34,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:34,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594992 2024-12-10T07:26:34,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:34,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:34,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594992) roll requested 2024-12-10T07:26:34,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815594998 2024-12-10T07:26:35,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594998 2024-12-10T07:26:35,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815594998) roll requested 2024-12-10T07:26:35,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595005 2024-12-10T07:26:35,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815594998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595005 2024-12-10T07:26:35,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595005) roll requested 2024-12-10T07:26:35,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595011 2024-12-10T07:26:35,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595011 2024-12-10T07:26:35,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595011) roll requested 2024-12-10T07:26:35,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595018 2024-12-10T07:26:35,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595018 2024-12-10T07:26:35,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595018) roll requested 2024-12-10T07:26:35,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595023 2024-12-10T07:26:35,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595023 2024-12-10T07:26:35,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595023) roll requested 2024-12-10T07:26:35,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595029 2024-12-10T07:26:35,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595029 2024-12-10T07:26:35,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595029) roll requested 2024-12-10T07:26:35,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595035 2024-12-10T07:26:35,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595035 2024-12-10T07:26:35,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595035) roll requested 2024-12-10T07:26:35,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595041 2024-12-10T07:26:35,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595041 2024-12-10T07:26:35,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595041) roll requested 2024-12-10T07:26:35,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595047 2024-12-10T07:26:35,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595047 2024-12-10T07:26:35,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595047) roll requested 2024-12-10T07:26:35,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595055 2024-12-10T07:26:35,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595055 2024-12-10T07:26:35,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595055) roll requested 2024-12-10T07:26:35,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595062 2024-12-10T07:26:35,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595062 2024-12-10T07:26:35,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595062) roll requested 2024-12-10T07:26:35,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595068 2024-12-10T07:26:35,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595068 2024-12-10T07:26:35,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595068) roll requested 2024-12-10T07:26:35,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595074 2024-12-10T07:26:35,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595074 2024-12-10T07:26:35,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595074) roll requested 2024-12-10T07:26:35,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595080 2024-12-10T07:26:35,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595080 2024-12-10T07:26:35,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595080) roll requested 2024-12-10T07:26:35,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595086 2024-12-10T07:26:35,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595086 2024-12-10T07:26:35,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595080 is not closed yet, will try archiving it next time 2024-12-10T07:26:35,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595086) roll requested 2024-12-10T07:26:35,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595091 2024-12-10T07:26:35,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595091 2024-12-10T07:26:35,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595091) roll requested 2024-12-10T07:26:35,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595155 2024-12-10T07:26:35,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595155 2024-12-10T07:26:35,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595155) roll requested 2024-12-10T07:26:35,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595161 2024-12-10T07:26:35,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595161 2024-12-10T07:26:35,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595161) roll requested 2024-12-10T07:26:35,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595168 2024-12-10T07:26:35,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595168 2024-12-10T07:26:35,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595168) roll requested 2024-12-10T07:26:35,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595173 2024-12-10T07:26:35,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595173 2024-12-10T07:26:35,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595173) roll requested 2024-12-10T07:26:35,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595180 2024-12-10T07:26:35,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595180 2024-12-10T07:26:35,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595180) roll requested 2024-12-10T07:26:35,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595187 2024-12-10T07:26:35,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595187 2024-12-10T07:26:35,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595187) roll requested 2024-12-10T07:26:35,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595193 2024-12-10T07:26:35,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595193 2024-12-10T07:26:35,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595193) roll requested 2024-12-10T07:26:35,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595201 2024-12-10T07:26:35,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595201 2024-12-10T07:26:35,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595201) roll requested 2024-12-10T07:26:35,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595207 2024-12-10T07:26:35,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595207 2024-12-10T07:26:35,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595207) roll requested 2024-12-10T07:26:35,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595215 2024-12-10T07:26:35,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595215 2024-12-10T07:26:35,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595215) roll requested 2024-12-10T07:26:35,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595221 2024-12-10T07:26:35,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595221 2024-12-10T07:26:35,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595221) roll requested 2024-12-10T07:26:35,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595226 2024-12-10T07:26:35,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595226 2024-12-10T07:26:35,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595226) roll requested 2024-12-10T07:26:35,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595231 2024-12-10T07:26:35,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595231 2024-12-10T07:26:35,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595231) roll requested 2024-12-10T07:26:35,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595238 2024-12-10T07:26:35,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595238 2024-12-10T07:26:35,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595238) roll requested 2024-12-10T07:26:35,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595242 2024-12-10T07:26:35,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595242 2024-12-10T07:26:35,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595242) roll requested 2024-12-10T07:26:35,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595251 2024-12-10T07:26:35,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595251 2024-12-10T07:26:35,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595251) roll requested 2024-12-10T07:26:35,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595262 2024-12-10T07:26:35,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595262 2024-12-10T07:26:35,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595262) roll requested 2024-12-10T07:26:35,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595270 2024-12-10T07:26:35,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595270 2024-12-10T07:26:35,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595270) roll requested 2024-12-10T07:26:35,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595276 2024-12-10T07:26:35,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595276 2024-12-10T07:26:35,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595276) roll requested 2024-12-10T07:26:35,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595281 2024-12-10T07:26:35,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595281 2024-12-10T07:26:35,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595281) roll requested 2024-12-10T07:26:35,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595286 2024-12-10T07:26:35,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595286 2024-12-10T07:26:35,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595286) roll requested 2024-12-10T07:26:35,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595292 2024-12-10T07:26:35,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595292 2024-12-10T07:26:35,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595292) roll requested 2024-12-10T07:26:35,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595298 2024-12-10T07:26:35,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595298 2024-12-10T07:26:35,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595298) roll requested 2024-12-10T07:26:35,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595304 2024-12-10T07:26:35,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595304 2024-12-10T07:26:35,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595304) roll requested 2024-12-10T07:26:35,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595310 2024-12-10T07:26:35,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595310 2024-12-10T07:26:35,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595310) roll requested 2024-12-10T07:26:35,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595317 2024-12-10T07:26:35,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595317 2024-12-10T07:26:35,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595317) roll requested 2024-12-10T07:26:35,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595322 2024-12-10T07:26:35,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595322 2024-12-10T07:26:35,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595322) roll requested 2024-12-10T07:26:35,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595327 2024-12-10T07:26:35,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595327 2024-12-10T07:26:35,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595327) roll requested 2024-12-10T07:26:35,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595333 2024-12-10T07:26:35,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595333 2024-12-10T07:26:35,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595333) roll requested 2024-12-10T07:26:35,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595340 2024-12-10T07:26:35,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595340 2024-12-10T07:26:35,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595340) roll requested 2024-12-10T07:26:35,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595347 2024-12-10T07:26:35,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595347 2024-12-10T07:26:35,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595347) roll requested 2024-12-10T07:26:35,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595355 2024-12-10T07:26:35,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595355 2024-12-10T07:26:35,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595355) roll requested 2024-12-10T07:26:35,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595363 2024-12-10T07:26:35,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595363 2024-12-10T07:26:35,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595363) roll requested 2024-12-10T07:26:35,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595372 2024-12-10T07:26:35,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595372 2024-12-10T07:26:35,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595372) roll requested 2024-12-10T07:26:35,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595380 2024-12-10T07:26:35,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595380 2024-12-10T07:26:35,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595380) roll requested 2024-12-10T07:26:35,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595388 2024-12-10T07:26:35,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595388 2024-12-10T07:26:35,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595388) roll requested 2024-12-10T07:26:35,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595397 2024-12-10T07:26:35,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595397 2024-12-10T07:26:35,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595397) roll requested 2024-12-10T07:26:35,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595402 2024-12-10T07:26:35,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595402 2024-12-10T07:26:35,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595402) roll requested 2024-12-10T07:26:35,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595407 2024-12-10T07:26:35,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595407 2024-12-10T07:26:35,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595407) roll requested 2024-12-10T07:26:35,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595415 2024-12-10T07:26:35,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595415 2024-12-10T07:26:35,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595415) roll requested 2024-12-10T07:26:35,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595422 2024-12-10T07:26:35,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595422 2024-12-10T07:26:35,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595422) roll requested 2024-12-10T07:26:35,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595432 2024-12-10T07:26:35,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595432 2024-12-10T07:26:35,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595432) roll requested 2024-12-10T07:26:35,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595439 2024-12-10T07:26:35,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595439 2024-12-10T07:26:35,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595439) roll requested 2024-12-10T07:26:35,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595445 2024-12-10T07:26:35,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595445 2024-12-10T07:26:35,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595445) roll requested 2024-12-10T07:26:35,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595451 2024-12-10T07:26:35,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595451 2024-12-10T07:26:35,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595451) roll requested 2024-12-10T07:26:35,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595461 2024-12-10T07:26:35,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595461 2024-12-10T07:26:35,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595461) roll requested 2024-12-10T07:26:35,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595466 2024-12-10T07:26:35,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595466 2024-12-10T07:26:35,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595466) roll requested 2024-12-10T07:26:35,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595474 2024-12-10T07:26:35,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595474 2024-12-10T07:26:35,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595474) roll requested 2024-12-10T07:26:35,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595479 2024-12-10T07:26:35,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595479 2024-12-10T07:26:35,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595479) roll requested 2024-12-10T07:26:35,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595484 2024-12-10T07:26:35,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595484 2024-12-10T07:26:35,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595484) roll requested 2024-12-10T07:26:35,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595489 2024-12-10T07:26:35,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595489 2024-12-10T07:26:35,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595489) roll requested 2024-12-10T07:26:35,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595495 2024-12-10T07:26:35,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595495 2024-12-10T07:26:35,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595495) roll requested 2024-12-10T07:26:35,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595502 2024-12-10T07:26:35,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595502 2024-12-10T07:26:35,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595502) roll requested 2024-12-10T07:26:35,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595510 2024-12-10T07:26:35,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595510 2024-12-10T07:26:35,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595510) roll requested 2024-12-10T07:26:35,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595519 2024-12-10T07:26:35,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595519 2024-12-10T07:26:35,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595519) roll requested 2024-12-10T07:26:35,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595529 2024-12-10T07:26:35,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595529 2024-12-10T07:26:35,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595529) roll requested 2024-12-10T07:26:35,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595534 2024-12-10T07:26:35,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595534 2024-12-10T07:26:35,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595534) roll requested 2024-12-10T07:26:35,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595542 2024-12-10T07:26:35,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595542 2024-12-10T07:26:35,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595542) roll requested 2024-12-10T07:26:35,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595547 2024-12-10T07:26:35,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595547 2024-12-10T07:26:35,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595547) roll requested 2024-12-10T07:26:35,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595552 2024-12-10T07:26:35,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595552 2024-12-10T07:26:35,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595552) roll requested 2024-12-10T07:26:35,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595558 2024-12-10T07:26:35,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595558 2024-12-10T07:26:35,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595558) roll requested 2024-12-10T07:26:35,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595566 2024-12-10T07:26:35,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595566 2024-12-10T07:26:35,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595566) roll requested 2024-12-10T07:26:35,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595571 2024-12-10T07:26:35,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595571 2024-12-10T07:26:35,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:35,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595571) roll requested 2024-12-10T07:26:35,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595579 2024-12-10T07:26:35,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595579 2024-12-10T07:26:35,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595579) roll requested 2024-12-10T07:26:35,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595587 2024-12-10T07:26:35,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595587 2024-12-10T07:26:35,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595587) roll requested 2024-12-10T07:26:35,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595592 2024-12-10T07:26:35,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595592 2024-12-10T07:26:35,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595592) roll requested 2024-12-10T07:26:35,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595599 2024-12-10T07:26:35,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595599 2024-12-10T07:26:35,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595599) roll requested 2024-12-10T07:26:35,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595607 2024-12-10T07:26:35,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595607 2024-12-10T07:26:35,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595607) roll requested 2024-12-10T07:26:35,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595617 2024-12-10T07:26:35,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595617 2024-12-10T07:26:35,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595617) roll requested 2024-12-10T07:26:35,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595623 2024-12-10T07:26:35,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595623 2024-12-10T07:26:35,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595623) roll requested 2024-12-10T07:26:35,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595629 2024-12-10T07:26:35,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595629 2024-12-10T07:26:35,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595629) roll requested 2024-12-10T07:26:35,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595639 2024-12-10T07:26:35,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595639 2024-12-10T07:26:35,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595639) roll requested 2024-12-10T07:26:35,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595644 2024-12-10T07:26:35,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595644 2024-12-10T07:26:35,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595644) roll requested 2024-12-10T07:26:35,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595650 2024-12-10T07:26:35,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595650 2024-12-10T07:26:35,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595650) roll requested 2024-12-10T07:26:35,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595656 2024-12-10T07:26:35,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595656 2024-12-10T07:26:35,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595656) roll requested 2024-12-10T07:26:35,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595661 2024-12-10T07:26:35,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595661 2024-12-10T07:26:35,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595661) roll requested 2024-12-10T07:26:35,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595667 2024-12-10T07:26:35,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595667 2024-12-10T07:26:35,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595667) roll requested 2024-12-10T07:26:35,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595673 2024-12-10T07:26:35,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595673 2024-12-10T07:26:35,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595673) roll requested 2024-12-10T07:26:35,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595678 2024-12-10T07:26:35,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595678 2024-12-10T07:26:35,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595678) roll requested 2024-12-10T07:26:35,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595683 2024-12-10T07:26:35,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595683 2024-12-10T07:26:35,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595683) roll requested 2024-12-10T07:26:35,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595689 2024-12-10T07:26:35,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595689 2024-12-10T07:26:35,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595689) roll requested 2024-12-10T07:26:35,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595696 2024-12-10T07:26:35,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595696 2024-12-10T07:26:35,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595696) roll requested 2024-12-10T07:26:35,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595704 2024-12-10T07:26:35,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595704 2024-12-10T07:26:35,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595704) roll requested 2024-12-10T07:26:35,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595711 2024-12-10T07:26:35,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595711 2024-12-10T07:26:35,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595711) roll requested 2024-12-10T07:26:35,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595719 2024-12-10T07:26:35,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595711 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595719 2024-12-10T07:26:35,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595719) roll requested 2024-12-10T07:26:35,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595728 2024-12-10T07:26:35,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595728 2024-12-10T07:26:35,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595728) roll requested 2024-12-10T07:26:35,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595737 2024-12-10T07:26:35,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595737 2024-12-10T07:26:35,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595737) roll requested 2024-12-10T07:26:35,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595745 2024-12-10T07:26:35,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595745 2024-12-10T07:26:35,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595745) roll requested 2024-12-10T07:26:35,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595751 2024-12-10T07:26:35,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595751 2024-12-10T07:26:35,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595751) roll requested 2024-12-10T07:26:35,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595756 2024-12-10T07:26:35,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595756 2024-12-10T07:26:35,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595756) roll requested 2024-12-10T07:26:35,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595761 2024-12-10T07:26:35,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595761 2024-12-10T07:26:35,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595761) roll requested 2024-12-10T07:26:35,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595768 2024-12-10T07:26:35,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595768 2024-12-10T07:26:35,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595768) roll requested 2024-12-10T07:26:35,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595773 2024-12-10T07:26:35,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595773 2024-12-10T07:26:35,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595773) roll requested 2024-12-10T07:26:35,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595779 2024-12-10T07:26:35,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595779 2024-12-10T07:26:35,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595779) roll requested 2024-12-10T07:26:35,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595784 2024-12-10T07:26:35,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595784 2024-12-10T07:26:35,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595784) roll requested 2024-12-10T07:26:35,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595789 2024-12-10T07:26:35,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595789 2024-12-10T07:26:35,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595789) roll requested 2024-12-10T07:26:35,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595796 2024-12-10T07:26:35,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595796 2024-12-10T07:26:35,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595796) roll requested 2024-12-10T07:26:35,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595801 2024-12-10T07:26:35,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595801 2024-12-10T07:26:35,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595801) roll requested 2024-12-10T07:26:35,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595808 2024-12-10T07:26:35,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595808 2024-12-10T07:26:35,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595808) roll requested 2024-12-10T07:26:35,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595815 2024-12-10T07:26:35,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595815 2024-12-10T07:26:35,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595815) roll requested 2024-12-10T07:26:35,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595823 2024-12-10T07:26:35,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595823 2024-12-10T07:26:35,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595823) roll requested 2024-12-10T07:26:35,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595831 2024-12-10T07:26:35,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595831 2024-12-10T07:26:35,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595831) roll requested 2024-12-10T07:26:35,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595840 2024-12-10T07:26:35,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595840 2024-12-10T07:26:35,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595840) roll requested 2024-12-10T07:26:35,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595846 2024-12-10T07:26:35,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595846 2024-12-10T07:26:35,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595846) roll requested 2024-12-10T07:26:35,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595854 2024-12-10T07:26:35,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595854 2024-12-10T07:26:35,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595854) roll requested 2024-12-10T07:26:35,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595864 2024-12-10T07:26:35,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595864 2024-12-10T07:26:35,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595864) roll requested 2024-12-10T07:26:35,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595873 2024-12-10T07:26:35,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595873 2024-12-10T07:26:35,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595873) roll requested 2024-12-10T07:26:35,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595878 2024-12-10T07:26:35,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595878 2024-12-10T07:26:35,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595878) roll requested 2024-12-10T07:26:35,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595888 2024-12-10T07:26:35,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595888 2024-12-10T07:26:35,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595888) roll requested 2024-12-10T07:26:35,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595897 2024-12-10T07:26:35,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595897 2024-12-10T07:26:35,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595897) roll requested 2024-12-10T07:26:35,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595904 2024-12-10T07:26:35,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595904 2024-12-10T07:26:35,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595904) roll requested 2024-12-10T07:26:35,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595910 2024-12-10T07:26:35,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595910 2024-12-10T07:26:35,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595910) roll requested 2024-12-10T07:26:35,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595915 2024-12-10T07:26:35,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595915 2024-12-10T07:26:35,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595915) roll requested 2024-12-10T07:26:35,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595920 2024-12-10T07:26:35,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595920 2024-12-10T07:26:35,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595920) roll requested 2024-12-10T07:26:35,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595927 2024-12-10T07:26:35,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595927 2024-12-10T07:26:35,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595927) roll requested 2024-12-10T07:26:35,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595933 2024-12-10T07:26:35,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595933 2024-12-10T07:26:35,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595933) roll requested 2024-12-10T07:26:35,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595941 2024-12-10T07:26:35,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595941 2024-12-10T07:26:35,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595941) roll requested 2024-12-10T07:26:35,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595949 2024-12-10T07:26:35,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595949 2024-12-10T07:26:35,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595949) roll requested 2024-12-10T07:26:35,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595956 2024-12-10T07:26:35,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595956 2024-12-10T07:26:35,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:35,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:35,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595956) roll requested 2024-12-10T07:26:35,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595965 2024-12-10T07:26:35,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595965 2024-12-10T07:26:35,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:35,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:35,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595965) roll requested 2024-12-10T07:26:35,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595974 2024-12-10T07:26:35,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595974 2024-12-10T07:26:35,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:35,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:35,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595974) roll requested 2024-12-10T07:26:35,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595983 2024-12-10T07:26:35,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:35,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595983 2024-12-10T07:26:35,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:35,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:35,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:35,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:35,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595983) roll requested 2024-12-10T07:26:35,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815595993 2024-12-10T07:26:36,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595993 2024-12-10T07:26:36,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815595993) roll requested 2024-12-10T07:26:36,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596003 2024-12-10T07:26:36,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815595993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596003 2024-12-10T07:26:36,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596003) roll requested 2024-12-10T07:26:36,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596012 2024-12-10T07:26:36,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596012 2024-12-10T07:26:36,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596012) roll requested 2024-12-10T07:26:36,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596017 2024-12-10T07:26:36,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596017 2024-12-10T07:26:36,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596017) roll requested 2024-12-10T07:26:36,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596025 2024-12-10T07:26:36,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596025 2024-12-10T07:26:36,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596025) roll requested 2024-12-10T07:26:36,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596035 2024-12-10T07:26:36,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596035 2024-12-10T07:26:36,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596035) roll requested 2024-12-10T07:26:36,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596040 2024-12-10T07:26:36,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596040 2024-12-10T07:26:36,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596040) roll requested 2024-12-10T07:26:36,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596045 2024-12-10T07:26:36,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596045 2024-12-10T07:26:36,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596045) roll requested 2024-12-10T07:26:36,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596053 2024-12-10T07:26:36,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596053 2024-12-10T07:26:36,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596053) roll requested 2024-12-10T07:26:36,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596061 2024-12-10T07:26:36,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596061 2024-12-10T07:26:36,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596061) roll requested 2024-12-10T07:26:36,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596070 2024-12-10T07:26:36,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596070 2024-12-10T07:26:36,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596070) roll requested 2024-12-10T07:26:36,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596077 2024-12-10T07:26:36,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596077 2024-12-10T07:26:36,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596077) roll requested 2024-12-10T07:26:36,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596086 2024-12-10T07:26:36,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596086 2024-12-10T07:26:36,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596086) roll requested 2024-12-10T07:26:36,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596096 2024-12-10T07:26:36,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596096 2024-12-10T07:26:36,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596096) roll requested 2024-12-10T07:26:36,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596104 2024-12-10T07:26:36,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596104 2024-12-10T07:26:36,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596104) roll requested 2024-12-10T07:26:36,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596109 2024-12-10T07:26:36,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596109 2024-12-10T07:26:36,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596109) roll requested 2024-12-10T07:26:36,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596114 2024-12-10T07:26:36,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596114 2024-12-10T07:26:36,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596114) roll requested 2024-12-10T07:26:36,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596119 2024-12-10T07:26:36,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596119 2024-12-10T07:26:36,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596119) roll requested 2024-12-10T07:26:36,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596124 2024-12-10T07:26:36,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596124 2024-12-10T07:26:36,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596124) roll requested 2024-12-10T07:26:36,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596130 2024-12-10T07:26:36,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596130 2024-12-10T07:26:36,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596130) roll requested 2024-12-10T07:26:36,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596137 2024-12-10T07:26:36,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596137 2024-12-10T07:26:36,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596137) roll requested 2024-12-10T07:26:36,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596144 2024-12-10T07:26:36,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596144 2024-12-10T07:26:36,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596144) roll requested 2024-12-10T07:26:36,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596151 2024-12-10T07:26:36,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596151 2024-12-10T07:26:36,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,158 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,158 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596151) roll requested 2024-12-10T07:26:36,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596158 2024-12-10T07:26:36,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596158 2024-12-10T07:26:36,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596158) roll requested 2024-12-10T07:26:36,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596167 2024-12-10T07:26:36,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596167 2024-12-10T07:26:36,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596167) roll requested 2024-12-10T07:26:36,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596177 2024-12-10T07:26:36,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596177 2024-12-10T07:26:36,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596177) roll requested 2024-12-10T07:26:36,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596185 2024-12-10T07:26:36,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596185 2024-12-10T07:26:36,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596185) roll requested 2024-12-10T07:26:36,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596193 2024-12-10T07:26:36,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596193 2024-12-10T07:26:36,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596193) roll requested 2024-12-10T07:26:36,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596201 2024-12-10T07:26:36,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596201 2024-12-10T07:26:36,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596201) roll requested 2024-12-10T07:26:36,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596209 2024-12-10T07:26:36,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596209 2024-12-10T07:26:36,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596209) roll requested 2024-12-10T07:26:36,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596221 2024-12-10T07:26:36,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596221 2024-12-10T07:26:36,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596221) roll requested 2024-12-10T07:26:36,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596229 2024-12-10T07:26:36,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596229 2024-12-10T07:26:36,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596229) roll requested 2024-12-10T07:26:36,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596234 2024-12-10T07:26:36,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596234 2024-12-10T07:26:36,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596234) roll requested 2024-12-10T07:26:36,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596239 2024-12-10T07:26:36,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596239 2024-12-10T07:26:36,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596239) roll requested 2024-12-10T07:26:36,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596244 2024-12-10T07:26:36,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596244 2024-12-10T07:26:36,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596244) roll requested 2024-12-10T07:26:36,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596249 2024-12-10T07:26:36,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596249 2024-12-10T07:26:36,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596249) roll requested 2024-12-10T07:26:36,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596256 2024-12-10T07:26:36,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596256 2024-12-10T07:26:36,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596256) roll requested 2024-12-10T07:26:36,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596263 2024-12-10T07:26:36,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596263 2024-12-10T07:26:36,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596263) roll requested 2024-12-10T07:26:36,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596269 2024-12-10T07:26:36,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596269 2024-12-10T07:26:36,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596269) roll requested 2024-12-10T07:26:36,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596274 2024-12-10T07:26:36,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596274 2024-12-10T07:26:36,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596274) roll requested 2024-12-10T07:26:36,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596281 2024-12-10T07:26:36,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596281 2024-12-10T07:26:36,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596281) roll requested 2024-12-10T07:26:36,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596288 2024-12-10T07:26:36,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596288 2024-12-10T07:26:36,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596288) roll requested 2024-12-10T07:26:36,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596296 2024-12-10T07:26:36,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596296 2024-12-10T07:26:36,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596296) roll requested 2024-12-10T07:26:36,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596305 2024-12-10T07:26:36,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596305 2024-12-10T07:26:36,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596305) roll requested 2024-12-10T07:26:36,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596314 2024-12-10T07:26:36,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596314 2024-12-10T07:26:36,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596314) roll requested 2024-12-10T07:26:36,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596322 2024-12-10T07:26:36,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596322 2024-12-10T07:26:36,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596322) roll requested 2024-12-10T07:26:36,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596329 2024-12-10T07:26:36,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596329 2024-12-10T07:26:36,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596329) roll requested 2024-12-10T07:26:36,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596337 2024-12-10T07:26:36,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596337 2024-12-10T07:26:36,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596337) roll requested 2024-12-10T07:26:36,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596344 2024-12-10T07:26:36,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596344 2024-12-10T07:26:36,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596344) roll requested 2024-12-10T07:26:36,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596352 2024-12-10T07:26:36,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596352 2024-12-10T07:26:36,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596352) roll requested 2024-12-10T07:26:36,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596360 2024-12-10T07:26:36,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596360 2024-12-10T07:26:36,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596360) roll requested 2024-12-10T07:26:36,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596368 2024-12-10T07:26:36,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596368 2024-12-10T07:26:36,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596368) roll requested 2024-12-10T07:26:36,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596376 2024-12-10T07:26:36,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596376 2024-12-10T07:26:36,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596376) roll requested 2024-12-10T07:26:36,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596385 2024-12-10T07:26:36,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596385 2024-12-10T07:26:36,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596385) roll requested 2024-12-10T07:26:36,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596394 2024-12-10T07:26:36,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596394 2024-12-10T07:26:36,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596394) roll requested 2024-12-10T07:26:36,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596400 2024-12-10T07:26:36,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596400 2024-12-10T07:26:36,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596400) roll requested 2024-12-10T07:26:36,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596407 2024-12-10T07:26:36,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596407 2024-12-10T07:26:36,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596407) roll requested 2024-12-10T07:26:36,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596415 2024-12-10T07:26:36,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596415 2024-12-10T07:26:36,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596415) roll requested 2024-12-10T07:26:36,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596425 2024-12-10T07:26:36,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596425 2024-12-10T07:26:36,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596425) roll requested 2024-12-10T07:26:36,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596434 2024-12-10T07:26:36,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596434 2024-12-10T07:26:36,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596434) roll requested 2024-12-10T07:26:36,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596443 2024-12-10T07:26:36,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596443 2024-12-10T07:26:36,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596443) roll requested 2024-12-10T07:26:36,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596453 2024-12-10T07:26:36,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596453 2024-12-10T07:26:36,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596453) roll requested 2024-12-10T07:26:36,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596465 2024-12-10T07:26:36,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596465 2024-12-10T07:26:36,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596465) roll requested 2024-12-10T07:26:36,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596472 2024-12-10T07:26:36,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596472 2024-12-10T07:26:36,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596472) roll requested 2024-12-10T07:26:36,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596478 2024-12-10T07:26:36,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596478 2024-12-10T07:26:36,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596478) roll requested 2024-12-10T07:26:36,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596485 2024-12-10T07:26:36,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596485 2024-12-10T07:26:36,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596485) roll requested 2024-12-10T07:26:36,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596490 2024-12-10T07:26:36,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596490 2024-12-10T07:26:36,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596490) roll requested 2024-12-10T07:26:36,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596497 2024-12-10T07:26:36,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596497 2024-12-10T07:26:36,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596497) roll requested 2024-12-10T07:26:36,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596507 2024-12-10T07:26:36,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596507 2024-12-10T07:26:36,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596507) roll requested 2024-12-10T07:26:36,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596513 2024-12-10T07:26:36,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596513 2024-12-10T07:26:36,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596513) roll requested 2024-12-10T07:26:36,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596519 2024-12-10T07:26:36,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596519 2024-12-10T07:26:36,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596519) roll requested 2024-12-10T07:26:36,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596524 2024-12-10T07:26:36,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596524 2024-12-10T07:26:36,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596524) roll requested 2024-12-10T07:26:36,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596529 2024-12-10T07:26:36,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596529 2024-12-10T07:26:36,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596529) roll requested 2024-12-10T07:26:36,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596537 2024-12-10T07:26:36,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596537 2024-12-10T07:26:36,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596537) roll requested 2024-12-10T07:26:36,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596543 2024-12-10T07:26:36,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596543 2024-12-10T07:26:36,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596543) roll requested 2024-12-10T07:26:36,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596548 2024-12-10T07:26:36,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596548 2024-12-10T07:26:36,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596548) roll requested 2024-12-10T07:26:36,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596553 2024-12-10T07:26:36,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596553 2024-12-10T07:26:36,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596553) roll requested 2024-12-10T07:26:36,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596559 2024-12-10T07:26:36,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596559 2024-12-10T07:26:36,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596559) roll requested 2024-12-10T07:26:36,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596566 2024-12-10T07:26:36,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596566 2024-12-10T07:26:36,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596566) roll requested 2024-12-10T07:26:36,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596572 2024-12-10T07:26:36,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596572 2024-12-10T07:26:36,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596572) roll requested 2024-12-10T07:26:36,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596578 2024-12-10T07:26:36,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596578 2024-12-10T07:26:36,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596578) roll requested 2024-12-10T07:26:36,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596584 2024-12-10T07:26:36,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596584 2024-12-10T07:26:36,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596584) roll requested 2024-12-10T07:26:36,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596589 2024-12-10T07:26:36,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596589 2024-12-10T07:26:36,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596589) roll requested 2024-12-10T07:26:36,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596597 2024-12-10T07:26:36,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596597 2024-12-10T07:26:36,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596597) roll requested 2024-12-10T07:26:36,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596604 2024-12-10T07:26:36,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596604 2024-12-10T07:26:36,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596604) roll requested 2024-12-10T07:26:36,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596611 2024-12-10T07:26:36,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596611 2024-12-10T07:26:36,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596611) roll requested 2024-12-10T07:26:36,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596616 2024-12-10T07:26:36,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596616 2024-12-10T07:26:36,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596616) roll requested 2024-12-10T07:26:36,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596620 2024-12-10T07:26:36,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596620 2024-12-10T07:26:36,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596620) roll requested 2024-12-10T07:26:36,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596627 2024-12-10T07:26:36,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596627 2024-12-10T07:26:36,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596627) roll requested 2024-12-10T07:26:36,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596637 2024-12-10T07:26:36,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596637 2024-12-10T07:26:36,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596637) roll requested 2024-12-10T07:26:36,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596642 2024-12-10T07:26:36,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596642 2024-12-10T07:26:36,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596642) roll requested 2024-12-10T07:26:36,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596646 2024-12-10T07:26:36,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596646 2024-12-10T07:26:36,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596646) roll requested 2024-12-10T07:26:36,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596651 2024-12-10T07:26:36,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596651 2024-12-10T07:26:36,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596651) roll requested 2024-12-10T07:26:36,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596658 2024-12-10T07:26:36,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596658 2024-12-10T07:26:36,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596658) roll requested 2024-12-10T07:26:36,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596665 2024-12-10T07:26:36,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596665 2024-12-10T07:26:36,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596665) roll requested 2024-12-10T07:26:36,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596673 2024-12-10T07:26:36,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596673 2024-12-10T07:26:36,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596673) roll requested 2024-12-10T07:26:36,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596681 2024-12-10T07:26:36,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596681 2024-12-10T07:26:36,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596681) roll requested 2024-12-10T07:26:36,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596687 2024-12-10T07:26:36,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596687 2024-12-10T07:26:36,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596687) roll requested 2024-12-10T07:26:36,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596692 2024-12-10T07:26:36,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596692 2024-12-10T07:26:36,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596692) roll requested 2024-12-10T07:26:36,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596700 2024-12-10T07:26:36,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596700 2024-12-10T07:26:36,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596700) roll requested 2024-12-10T07:26:36,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596707 2024-12-10T07:26:36,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596707 2024-12-10T07:26:36,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596707) roll requested 2024-12-10T07:26:36,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596714 2024-12-10T07:26:36,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596714 2024-12-10T07:26:36,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596714) roll requested 2024-12-10T07:26:36,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596719 2024-12-10T07:26:36,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596719 2024-12-10T07:26:36,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596719) roll requested 2024-12-10T07:26:36,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596726 2024-12-10T07:26:36,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596726 2024-12-10T07:26:36,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596726) roll requested 2024-12-10T07:26:36,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596731 2024-12-10T07:26:36,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596731 2024-12-10T07:26:36,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596731) roll requested 2024-12-10T07:26:36,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596736 2024-12-10T07:26:36,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596736 2024-12-10T07:26:36,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596736) roll requested 2024-12-10T07:26:36,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596742 2024-12-10T07:26:36,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596742 2024-12-10T07:26:36,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596742) roll requested 2024-12-10T07:26:36,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596748 2024-12-10T07:26:36,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596748 2024-12-10T07:26:36,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596748) roll requested 2024-12-10T07:26:36,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596756 2024-12-10T07:26:36,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596756 2024-12-10T07:26:36,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596756) roll requested 2024-12-10T07:26:36,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596766 2024-12-10T07:26:36,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596766 2024-12-10T07:26:36,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596766) roll requested 2024-12-10T07:26:36,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596775 2024-12-10T07:26:36,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596775 2024-12-10T07:26:36,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596775) roll requested 2024-12-10T07:26:36,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596784 2024-12-10T07:26:36,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596784 2024-12-10T07:26:36,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596784) roll requested 2024-12-10T07:26:36,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596791 2024-12-10T07:26:36,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596791 2024-12-10T07:26:36,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596791) roll requested 2024-12-10T07:26:36,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596798 2024-12-10T07:26:36,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596798 2024-12-10T07:26:36,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596798) roll requested 2024-12-10T07:26:36,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596804 2024-12-10T07:26:36,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596804 2024-12-10T07:26:36,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596804) roll requested 2024-12-10T07:26:36,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596812 2024-12-10T07:26:36,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596812 2024-12-10T07:26:36,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596812) roll requested 2024-12-10T07:26:36,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596819 2024-12-10T07:26:36,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596819 2024-12-10T07:26:36,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596819) roll requested 2024-12-10T07:26:36,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596827 2024-12-10T07:26:36,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596827 2024-12-10T07:26:36,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596827) roll requested 2024-12-10T07:26:36,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596837 2024-12-10T07:26:36,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596837 2024-12-10T07:26:36,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596837) roll requested 2024-12-10T07:26:36,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596845 2024-12-10T07:26:36,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596845 2024-12-10T07:26:36,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596845) roll requested 2024-12-10T07:26:36,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596852 2024-12-10T07:26:36,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596852 2024-12-10T07:26:36,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596852) roll requested 2024-12-10T07:26:36,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596860 2024-12-10T07:26:36,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596860 2024-12-10T07:26:36,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596860) roll requested 2024-12-10T07:26:36,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596868 2024-12-10T07:26:36,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596868 2024-12-10T07:26:36,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596868) roll requested 2024-12-10T07:26:36,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596874 2024-12-10T07:26:36,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596874 2024-12-10T07:26:36,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596874) roll requested 2024-12-10T07:26:36,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596882 2024-12-10T07:26:36,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596882 2024-12-10T07:26:36,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596882) roll requested 2024-12-10T07:26:36,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596891 2024-12-10T07:26:36,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596891 2024-12-10T07:26:36,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596891) roll requested 2024-12-10T07:26:36,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596901 2024-12-10T07:26:36,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596901 2024-12-10T07:26:36,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596901) roll requested 2024-12-10T07:26:36,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596909 2024-12-10T07:26:36,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596909 2024-12-10T07:26:36,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596909) roll requested 2024-12-10T07:26:36,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596917 2024-12-10T07:26:36,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596917 2024-12-10T07:26:36,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596917) roll requested 2024-12-10T07:26:36,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596927 2024-12-10T07:26:36,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596927 2024-12-10T07:26:36,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596927) roll requested 2024-12-10T07:26:36,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596932 2024-12-10T07:26:36,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596932 2024-12-10T07:26:36,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596932) roll requested 2024-12-10T07:26:36,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596942 2024-12-10T07:26:36,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596942 2024-12-10T07:26:36,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596942) roll requested 2024-12-10T07:26:36,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596950 2024-12-10T07:26:36,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596950 2024-12-10T07:26:36,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596950) roll requested 2024-12-10T07:26:36,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596956 2024-12-10T07:26:36,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596956 2024-12-10T07:26:36,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596956) roll requested 2024-12-10T07:26:36,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596961 2024-12-10T07:26:36,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596961 2024-12-10T07:26:36,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596961) roll requested 2024-12-10T07:26:36,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596967 2024-12-10T07:26:36,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596967 2024-12-10T07:26:36,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596967) roll requested 2024-12-10T07:26:36,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596974 2024-12-10T07:26:36,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596974 2024-12-10T07:26:36,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596974) roll requested 2024-12-10T07:26:36,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596984 2024-12-10T07:26:36,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596984 2024-12-10T07:26:36,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596984) roll requested 2024-12-10T07:26:36,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596990 2024-12-10T07:26:36,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596990 2024-12-10T07:26:36,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:36,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:36,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:36,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:36,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596990) roll requested 2024-12-10T07:26:36,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815596995 2024-12-10T07:26:36,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:36,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596995 2024-12-10T07:26:36,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815596995) roll requested 2024-12-10T07:26:37,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597000 2024-12-10T07:26:37,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815596995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597000 2024-12-10T07:26:37,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597000) roll requested 2024-12-10T07:26:37,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597007 2024-12-10T07:26:37,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597007 2024-12-10T07:26:37,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597007) roll requested 2024-12-10T07:26:37,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597014 2024-12-10T07:26:37,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597014 2024-12-10T07:26:37,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597014) roll requested 2024-12-10T07:26:37,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597023 2024-12-10T07:26:37,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597023 2024-12-10T07:26:37,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597023) roll requested 2024-12-10T07:26:37,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597032 2024-12-10T07:26:37,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597032 2024-12-10T07:26:37,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597032) roll requested 2024-12-10T07:26:37,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597040 2024-12-10T07:26:37,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597040 2024-12-10T07:26:37,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597040) roll requested 2024-12-10T07:26:37,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597048 2024-12-10T07:26:37,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597048 2024-12-10T07:26:37,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597048) roll requested 2024-12-10T07:26:37,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597055 2024-12-10T07:26:37,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597055 2024-12-10T07:26:37,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597055) roll requested 2024-12-10T07:26:37,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597063 2024-12-10T07:26:37,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597063 2024-12-10T07:26:37,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597063) roll requested 2024-12-10T07:26:37,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597073 2024-12-10T07:26:37,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597073 2024-12-10T07:26:37,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597073) roll requested 2024-12-10T07:26:37,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597078 2024-12-10T07:26:37,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597078 2024-12-10T07:26:37,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597078) roll requested 2024-12-10T07:26:37,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597083 2024-12-10T07:26:37,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597083 2024-12-10T07:26:37,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597083) roll requested 2024-12-10T07:26:37,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597089 2024-12-10T07:26:37,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597089 2024-12-10T07:26:37,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597089) roll requested 2024-12-10T07:26:37,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597095 2024-12-10T07:26:37,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597095 2024-12-10T07:26:37,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597095) roll requested 2024-12-10T07:26:37,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597103 2024-12-10T07:26:37,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597103 2024-12-10T07:26:37,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597103) roll requested 2024-12-10T07:26:37,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597112 2024-12-10T07:26:37,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597112 2024-12-10T07:26:37,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597112) roll requested 2024-12-10T07:26:37,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597119 2024-12-10T07:26:37,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597119 2024-12-10T07:26:37,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597119) roll requested 2024-12-10T07:26:37,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597123 2024-12-10T07:26:37,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597123 2024-12-10T07:26:37,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597123) roll requested 2024-12-10T07:26:37,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597128 2024-12-10T07:26:37,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597128 2024-12-10T07:26:37,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597128) roll requested 2024-12-10T07:26:37,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597153 2024-12-10T07:26:37,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597153 2024-12-10T07:26:37,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597153) roll requested 2024-12-10T07:26:37,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597162 2024-12-10T07:26:37,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597162 2024-12-10T07:26:37,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597162) roll requested 2024-12-10T07:26:37,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597168 2024-12-10T07:26:37,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597168 2024-12-10T07:26:37,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597168) roll requested 2024-12-10T07:26:37,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597173 2024-12-10T07:26:37,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597173 2024-12-10T07:26:37,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597173) roll requested 2024-12-10T07:26:37,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597180 2024-12-10T07:26:37,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597180 2024-12-10T07:26:37,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597180) roll requested 2024-12-10T07:26:37,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597187 2024-12-10T07:26:37,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597187 2024-12-10T07:26:37,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597187) roll requested 2024-12-10T07:26:37,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597196 2024-12-10T07:26:37,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597196 2024-12-10T07:26:37,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597196) roll requested 2024-12-10T07:26:37,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597201 2024-12-10T07:26:37,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597201 2024-12-10T07:26:37,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597201) roll requested 2024-12-10T07:26:37,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597206 2024-12-10T07:26:37,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597206 2024-12-10T07:26:37,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597206) roll requested 2024-12-10T07:26:37,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597210 2024-12-10T07:26:37,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597210 2024-12-10T07:26:37,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597210) roll requested 2024-12-10T07:26:37,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597218 2024-12-10T07:26:37,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597218 2024-12-10T07:26:37,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597218) roll requested 2024-12-10T07:26:37,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597225 2024-12-10T07:26:37,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597225 2024-12-10T07:26:37,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597225) roll requested 2024-12-10T07:26:37,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597232 2024-12-10T07:26:37,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597232 2024-12-10T07:26:37,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597232) roll requested 2024-12-10T07:26:37,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597240 2024-12-10T07:26:37,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597240 2024-12-10T07:26:37,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597240) roll requested 2024-12-10T07:26:37,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597248 2024-12-10T07:26:37,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597248 2024-12-10T07:26:37,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597248) roll requested 2024-12-10T07:26:37,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597256 2024-12-10T07:26:37,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597256 2024-12-10T07:26:37,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597256) roll requested 2024-12-10T07:26:37,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597262 2024-12-10T07:26:37,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597262 2024-12-10T07:26:37,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597262) roll requested 2024-12-10T07:26:37,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597271 2024-12-10T07:26:37,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597271 2024-12-10T07:26:37,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597271) roll requested 2024-12-10T07:26:37,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597280 2024-12-10T07:26:37,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597280 2024-12-10T07:26:37,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597280) roll requested 2024-12-10T07:26:37,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597290 2024-12-10T07:26:37,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597290 2024-12-10T07:26:37,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597290) roll requested 2024-12-10T07:26:37,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597296 2024-12-10T07:26:37,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597296 2024-12-10T07:26:37,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597296) roll requested 2024-12-10T07:26:37,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597301 2024-12-10T07:26:37,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597301 2024-12-10T07:26:37,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597301) roll requested 2024-12-10T07:26:37,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597306 2024-12-10T07:26:37,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597306 2024-12-10T07:26:37,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597306) roll requested 2024-12-10T07:26:37,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597312 2024-12-10T07:26:37,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597312 2024-12-10T07:26:37,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597312) roll requested 2024-12-10T07:26:37,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597321 2024-12-10T07:26:37,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597321 2024-12-10T07:26:37,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597321) roll requested 2024-12-10T07:26:37,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597328 2024-12-10T07:26:37,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597328 2024-12-10T07:26:37,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597328) roll requested 2024-12-10T07:26:37,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597335 2024-12-10T07:26:37,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597335 2024-12-10T07:26:37,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597335) roll requested 2024-12-10T07:26:37,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597343 2024-12-10T07:26:37,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597343 2024-12-10T07:26:37,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597343) roll requested 2024-12-10T07:26:37,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597352 2024-12-10T07:26:37,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597352 2024-12-10T07:26:37,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597352) roll requested 2024-12-10T07:26:37,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597361 2024-12-10T07:26:37,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597361 2024-12-10T07:26:37,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597361) roll requested 2024-12-10T07:26:37,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597370 2024-12-10T07:26:37,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597370 2024-12-10T07:26:37,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597370) roll requested 2024-12-10T07:26:37,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597378 2024-12-10T07:26:37,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597378 2024-12-10T07:26:37,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597378) roll requested 2024-12-10T07:26:37,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597386 2024-12-10T07:26:37,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597386 2024-12-10T07:26:37,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597386) roll requested 2024-12-10T07:26:37,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597393 2024-12-10T07:26:37,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597393 2024-12-10T07:26:37,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597393) roll requested 2024-12-10T07:26:37,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597400 2024-12-10T07:26:37,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597400 2024-12-10T07:26:37,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597400) roll requested 2024-12-10T07:26:37,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597408 2024-12-10T07:26:37,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597408 2024-12-10T07:26:37,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597408) roll requested 2024-12-10T07:26:37,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597416 2024-12-10T07:26:37,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597416 2024-12-10T07:26:37,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597416) roll requested 2024-12-10T07:26:37,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597423 2024-12-10T07:26:37,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597423 2024-12-10T07:26:37,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597423) roll requested 2024-12-10T07:26:37,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597431 2024-12-10T07:26:37,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597431 2024-12-10T07:26:37,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597431) roll requested 2024-12-10T07:26:37,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597440 2024-12-10T07:26:37,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597440 2024-12-10T07:26:37,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597440) roll requested 2024-12-10T07:26:37,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597448 2024-12-10T07:26:37,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597448 2024-12-10T07:26:37,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597448) roll requested 2024-12-10T07:26:37,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597457 2024-12-10T07:26:37,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597457 2024-12-10T07:26:37,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597457) roll requested 2024-12-10T07:26:37,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597468 2024-12-10T07:26:37,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597468 2024-12-10T07:26:37,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597468) roll requested 2024-12-10T07:26:37,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597475 2024-12-10T07:26:37,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597475 2024-12-10T07:26:37,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597475) roll requested 2024-12-10T07:26:37,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597484 2024-12-10T07:26:37,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597484 2024-12-10T07:26:37,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597484) roll requested 2024-12-10T07:26:37,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597494 2024-12-10T07:26:37,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597494 2024-12-10T07:26:37,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597494) roll requested 2024-12-10T07:26:37,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597501 2024-12-10T07:26:37,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597501 2024-12-10T07:26:37,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597501) roll requested 2024-12-10T07:26:37,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597509 2024-12-10T07:26:37,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597509 2024-12-10T07:26:37,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597509) roll requested 2024-12-10T07:26:37,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597519 2024-12-10T07:26:37,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597519 2024-12-10T07:26:37,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597519) roll requested 2024-12-10T07:26:37,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597526 2024-12-10T07:26:37,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597526 2024-12-10T07:26:37,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597519 is not closed yet, will try archiving it next time 2024-12-10T07:26:37,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597526) roll requested 2024-12-10T07:26:37,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597533 2024-12-10T07:26:37,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597533 2024-12-10T07:26:37,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597533) roll requested 2024-12-10T07:26:37,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597542 2024-12-10T07:26:37,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597542 2024-12-10T07:26:37,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597542) roll requested 2024-12-10T07:26:37,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597550 2024-12-10T07:26:37,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597550 2024-12-10T07:26:37,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597550) roll requested 2024-12-10T07:26:37,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597559 2024-12-10T07:26:37,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597559 2024-12-10T07:26:37,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597559) roll requested 2024-12-10T07:26:37,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597571 2024-12-10T07:26:37,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597571 2024-12-10T07:26:37,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597571) roll requested 2024-12-10T07:26:37,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597576 2024-12-10T07:26:37,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597576 2024-12-10T07:26:37,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597576) roll requested 2024-12-10T07:26:37,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597581 2024-12-10T07:26:37,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597581 2024-12-10T07:26:37,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597581) roll requested 2024-12-10T07:26:37,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597587 2024-12-10T07:26:37,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597587 2024-12-10T07:26:37,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597587) roll requested 2024-12-10T07:26:37,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597595 2024-12-10T07:26:37,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597595 2024-12-10T07:26:37,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597595) roll requested 2024-12-10T07:26:37,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597603 2024-12-10T07:26:37,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597603 2024-12-10T07:26:37,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597603) roll requested 2024-12-10T07:26:37,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597614 2024-12-10T07:26:37,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597614 2024-12-10T07:26:37,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597614) roll requested 2024-12-10T07:26:37,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597624 2024-12-10T07:26:37,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597624 2024-12-10T07:26:37,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597624) roll requested 2024-12-10T07:26:37,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597633 2024-12-10T07:26:37,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597633 2024-12-10T07:26:37,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597633) roll requested 2024-12-10T07:26:37,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597642 2024-12-10T07:26:37,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597642 2024-12-10T07:26:37,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597642) roll requested 2024-12-10T07:26:37,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597649 2024-12-10T07:26:37,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597649 2024-12-10T07:26:37,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597649) roll requested 2024-12-10T07:26:37,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597656 2024-12-10T07:26:37,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597656 2024-12-10T07:26:37,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597656) roll requested 2024-12-10T07:26:37,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597664 2024-12-10T07:26:37,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597664 2024-12-10T07:26:37,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597664) roll requested 2024-12-10T07:26:37,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597674 2024-12-10T07:26:37,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597674 2024-12-10T07:26:37,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597674) roll requested 2024-12-10T07:26:37,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597684 2024-12-10T07:26:37,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597684 2024-12-10T07:26:37,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597684) roll requested 2024-12-10T07:26:37,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597694 2024-12-10T07:26:37,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597694 2024-12-10T07:26:37,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597694) roll requested 2024-12-10T07:26:37,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597703 2024-12-10T07:26:37,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597703 2024-12-10T07:26:37,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,711 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597703) roll requested 2024-12-10T07:26:37,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597711 2024-12-10T07:26:37,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597711 2024-12-10T07:26:37,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597711) roll requested 2024-12-10T07:26:37,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597720 2024-12-10T07:26:37,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597711 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597720 2024-12-10T07:26:37,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597720) roll requested 2024-12-10T07:26:37,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597728 2024-12-10T07:26:37,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597728 2024-12-10T07:26:37,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597728) roll requested 2024-12-10T07:26:37,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597734 2024-12-10T07:26:37,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597734 2024-12-10T07:26:37,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597734) roll requested 2024-12-10T07:26:37,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597743 2024-12-10T07:26:37,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597743 2024-12-10T07:26:37,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597743) roll requested 2024-12-10T07:26:37,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597752 2024-12-10T07:26:37,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597752 2024-12-10T07:26:37,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597752) roll requested 2024-12-10T07:26:37,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597760 2024-12-10T07:26:37,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597760 2024-12-10T07:26:37,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597760) roll requested 2024-12-10T07:26:37,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597768 2024-12-10T07:26:37,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597768 2024-12-10T07:26:37,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597768) roll requested 2024-12-10T07:26:37,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597777 2024-12-10T07:26:37,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597777 2024-12-10T07:26:37,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597777) roll requested 2024-12-10T07:26:37,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597787 2024-12-10T07:26:37,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597787 2024-12-10T07:26:37,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597787) roll requested 2024-12-10T07:26:37,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597794 2024-12-10T07:26:37,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597794 2024-12-10T07:26:37,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597794) roll requested 2024-12-10T07:26:37,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597802 2024-12-10T07:26:37,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597802 2024-12-10T07:26:37,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597802) roll requested 2024-12-10T07:26:37,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597811 2024-12-10T07:26:37,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597811 2024-12-10T07:26:37,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597811) roll requested 2024-12-10T07:26:37,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597820 2024-12-10T07:26:37,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597820 2024-12-10T07:26:37,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597820) roll requested 2024-12-10T07:26:37,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597827 2024-12-10T07:26:37,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597827 2024-12-10T07:26:37,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597827) roll requested 2024-12-10T07:26:37,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597834 2024-12-10T07:26:37,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597834 2024-12-10T07:26:37,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597834) roll requested 2024-12-10T07:26:37,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597842 2024-12-10T07:26:37,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597842 2024-12-10T07:26:37,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597842) roll requested 2024-12-10T07:26:37,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597849 2024-12-10T07:26:37,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597849 2024-12-10T07:26:37,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597849) roll requested 2024-12-10T07:26:37,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597857 2024-12-10T07:26:37,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597857 2024-12-10T07:26:37,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597857) roll requested 2024-12-10T07:26:37,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597866 2024-12-10T07:26:37,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597866 2024-12-10T07:26:37,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597866) roll requested 2024-12-10T07:26:37,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597873 2024-12-10T07:26:37,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597873 2024-12-10T07:26:37,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597873) roll requested 2024-12-10T07:26:37,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597883 2024-12-10T07:26:37,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597883 2024-12-10T07:26:37,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597883) roll requested 2024-12-10T07:26:37,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597892 2024-12-10T07:26:37,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597892 2024-12-10T07:26:37,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597892) roll requested 2024-12-10T07:26:37,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597897 2024-12-10T07:26:37,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597897 2024-12-10T07:26:37,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597897) roll requested 2024-12-10T07:26:37,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597903 2024-12-10T07:26:37,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597903 2024-12-10T07:26:37,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597903) roll requested 2024-12-10T07:26:37,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597910 2024-12-10T07:26:37,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597910 2024-12-10T07:26:37,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597910) roll requested 2024-12-10T07:26:37,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597919 2024-12-10T07:26:37,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597919 2024-12-10T07:26:37,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597919) roll requested 2024-12-10T07:26:37,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597927 2024-12-10T07:26:37,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597927 2024-12-10T07:26:37,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597927) roll requested 2024-12-10T07:26:37,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597934 2024-12-10T07:26:37,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597934 2024-12-10T07:26:37,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597934) roll requested 2024-12-10T07:26:37,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597944 2024-12-10T07:26:37,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597944 2024-12-10T07:26:37,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597944) roll requested 2024-12-10T07:26:37,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597954 2024-12-10T07:26:37,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597954 2024-12-10T07:26:37,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597954) roll requested 2024-12-10T07:26:37,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597962 2024-12-10T07:26:37,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597962 2024-12-10T07:26:37,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597962) roll requested 2024-12-10T07:26:37,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597970 2024-12-10T07:26:37,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597970 2024-12-10T07:26:37,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597970) roll requested 2024-12-10T07:26:37,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597978 2024-12-10T07:26:37,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597978 2024-12-10T07:26:37,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597978) roll requested 2024-12-10T07:26:37,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597988 2024-12-10T07:26:37,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:37,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597988 2024-12-10T07:26:37,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:37,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:37,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:37,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:37,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597988) roll requested 2024-12-10T07:26:37,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815597997 2024-12-10T07:26:38,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597997 2024-12-10T07:26:38,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815597997) roll requested 2024-12-10T07:26:38,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598004 2024-12-10T07:26:38,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815597997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598004 2024-12-10T07:26:38,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598004) roll requested 2024-12-10T07:26:38,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598012 2024-12-10T07:26:38,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598012 2024-12-10T07:26:38,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598012) roll requested 2024-12-10T07:26:38,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598020 2024-12-10T07:26:38,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598020 2024-12-10T07:26:38,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598020) roll requested 2024-12-10T07:26:38,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598029 2024-12-10T07:26:38,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598029 2024-12-10T07:26:38,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598029) roll requested 2024-12-10T07:26:38,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598037 2024-12-10T07:26:38,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598037 2024-12-10T07:26:38,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598037) roll requested 2024-12-10T07:26:38,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598047 2024-12-10T07:26:38,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598047 2024-12-10T07:26:38,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598047) roll requested 2024-12-10T07:26:38,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598057 2024-12-10T07:26:38,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598057 2024-12-10T07:26:38,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598057) roll requested 2024-12-10T07:26:38,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598065 2024-12-10T07:26:38,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598065 2024-12-10T07:26:38,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598065) roll requested 2024-12-10T07:26:38,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598072 2024-12-10T07:26:38,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598072 2024-12-10T07:26:38,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598072) roll requested 2024-12-10T07:26:38,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598081 2024-12-10T07:26:38,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598081 2024-12-10T07:26:38,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598081) roll requested 2024-12-10T07:26:38,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598090 2024-12-10T07:26:38,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598090 2024-12-10T07:26:38,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598090) roll requested 2024-12-10T07:26:38,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598097 2024-12-10T07:26:38,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598097 2024-12-10T07:26:38,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598097) roll requested 2024-12-10T07:26:38,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598104 2024-12-10T07:26:38,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598104 2024-12-10T07:26:38,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598104) roll requested 2024-12-10T07:26:38,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598109 2024-12-10T07:26:38,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598109 2024-12-10T07:26:38,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598109) roll requested 2024-12-10T07:26:38,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598113 2024-12-10T07:26:38,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598113 2024-12-10T07:26:38,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598113) roll requested 2024-12-10T07:26:38,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598119 2024-12-10T07:26:38,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598119 2024-12-10T07:26:38,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598119) roll requested 2024-12-10T07:26:38,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598125 2024-12-10T07:26:38,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598125 2024-12-10T07:26:38,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598125) roll requested 2024-12-10T07:26:38,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598132 2024-12-10T07:26:38,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598132 2024-12-10T07:26:38,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598132) roll requested 2024-12-10T07:26:38,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598140 2024-12-10T07:26:38,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598140 2024-12-10T07:26:38,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598140) roll requested 2024-12-10T07:26:38,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598148 2024-12-10T07:26:38,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598148 2024-12-10T07:26:38,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598148) roll requested 2024-12-10T07:26:38,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598155 2024-12-10T07:26:38,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598155 2024-12-10T07:26:38,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598155) roll requested 2024-12-10T07:26:38,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598162 2024-12-10T07:26:38,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598162 2024-12-10T07:26:38,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598162) roll requested 2024-12-10T07:26:38,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598171 2024-12-10T07:26:38,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598171 2024-12-10T07:26:38,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598171) roll requested 2024-12-10T07:26:38,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598177 2024-12-10T07:26:38,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598177 2024-12-10T07:26:38,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598177) roll requested 2024-12-10T07:26:38,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598182 2024-12-10T07:26:38,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598182 2024-12-10T07:26:38,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598182) roll requested 2024-12-10T07:26:38,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598190 2024-12-10T07:26:38,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598190 2024-12-10T07:26:38,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598190) roll requested 2024-12-10T07:26:38,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598198 2024-12-10T07:26:38,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598198 2024-12-10T07:26:38,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598198) roll requested 2024-12-10T07:26:38,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598205 2024-12-10T07:26:38,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598205 2024-12-10T07:26:38,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598205) roll requested 2024-12-10T07:26:38,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598214 2024-12-10T07:26:38,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598214 2024-12-10T07:26:38,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598214) roll requested 2024-12-10T07:26:38,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598219 2024-12-10T07:26:38,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598219 2024-12-10T07:26:38,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598219) roll requested 2024-12-10T07:26:38,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598225 2024-12-10T07:26:38,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598225 2024-12-10T07:26:38,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598225) roll requested 2024-12-10T07:26:38,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598232 2024-12-10T07:26:38,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598232 2024-12-10T07:26:38,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598232) roll requested 2024-12-10T07:26:38,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598240 2024-12-10T07:26:38,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598240 2024-12-10T07:26:38,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598240) roll requested 2024-12-10T07:26:38,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598248 2024-12-10T07:26:38,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598248 2024-12-10T07:26:38,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598248) roll requested 2024-12-10T07:26:38,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598258 2024-12-10T07:26:38,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598258 2024-12-10T07:26:38,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598258) roll requested 2024-12-10T07:26:38,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598267 2024-12-10T07:26:38,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598267 2024-12-10T07:26:38,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598267) roll requested 2024-12-10T07:26:38,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598274 2024-12-10T07:26:38,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598274 2024-12-10T07:26:38,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598274) roll requested 2024-12-10T07:26:38,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598283 2024-12-10T07:26:38,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598283 2024-12-10T07:26:38,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598283) roll requested 2024-12-10T07:26:38,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598296 2024-12-10T07:26:38,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598296 2024-12-10T07:26:38,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598296) roll requested 2024-12-10T07:26:38,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598302 2024-12-10T07:26:38,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598302 2024-12-10T07:26:38,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598302) roll requested 2024-12-10T07:26:38,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598308 2024-12-10T07:26:38,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598308 2024-12-10T07:26:38,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598308) roll requested 2024-12-10T07:26:38,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598316 2024-12-10T07:26:38,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598316 2024-12-10T07:26:38,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598316) roll requested 2024-12-10T07:26:38,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598324 2024-12-10T07:26:38,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598324 2024-12-10T07:26:38,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598324) roll requested 2024-12-10T07:26:38,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598332 2024-12-10T07:26:38,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598332 2024-12-10T07:26:38,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598332) roll requested 2024-12-10T07:26:38,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598337 2024-12-10T07:26:38,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598337 2024-12-10T07:26:38,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598337) roll requested 2024-12-10T07:26:38,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598344 2024-12-10T07:26:38,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598344 2024-12-10T07:26:38,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598344) roll requested 2024-12-10T07:26:38,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598352 2024-12-10T07:26:38,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598352 2024-12-10T07:26:38,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598352) roll requested 2024-12-10T07:26:38,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598359 2024-12-10T07:26:38,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598359 2024-12-10T07:26:38,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598359) roll requested 2024-12-10T07:26:38,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598367 2024-12-10T07:26:38,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598367 2024-12-10T07:26:38,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598367) roll requested 2024-12-10T07:26:38,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598375 2024-12-10T07:26:38,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598375 2024-12-10T07:26:38,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598375) roll requested 2024-12-10T07:26:38,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598381 2024-12-10T07:26:38,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598381 2024-12-10T07:26:38,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598381) roll requested 2024-12-10T07:26:38,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598389 2024-12-10T07:26:38,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598389 2024-12-10T07:26:38,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598389) roll requested 2024-12-10T07:26:38,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598396 2024-12-10T07:26:38,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598396 2024-12-10T07:26:38,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598396) roll requested 2024-12-10T07:26:38,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598406 2024-12-10T07:26:38,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598406 2024-12-10T07:26:38,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598406) roll requested 2024-12-10T07:26:38,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598414 2024-12-10T07:26:38,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598414 2024-12-10T07:26:38,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598414) roll requested 2024-12-10T07:26:38,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598423 2024-12-10T07:26:38,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598423 2024-12-10T07:26:38,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598423) roll requested 2024-12-10T07:26:38,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598430 2024-12-10T07:26:38,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598430 2024-12-10T07:26:38,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598430) roll requested 2024-12-10T07:26:38,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598439 2024-12-10T07:26:38,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598439 2024-12-10T07:26:38,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,447 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598439) roll requested 2024-12-10T07:26:38,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598448 2024-12-10T07:26:38,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598448 2024-12-10T07:26:38,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598448) roll requested 2024-12-10T07:26:38,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598457 2024-12-10T07:26:38,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598457 2024-12-10T07:26:38,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598457) roll requested 2024-12-10T07:26:38,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598466 2024-12-10T07:26:38,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598466 2024-12-10T07:26:38,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598466) roll requested 2024-12-10T07:26:38,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598472 2024-12-10T07:26:38,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598472 2024-12-10T07:26:38,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598472) roll requested 2024-12-10T07:26:38,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598477 2024-12-10T07:26:38,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598477 2024-12-10T07:26:38,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598477) roll requested 2024-12-10T07:26:38,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598484 2024-12-10T07:26:38,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598484 2024-12-10T07:26:38,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598484) roll requested 2024-12-10T07:26:38,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598491 2024-12-10T07:26:38,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598491 2024-12-10T07:26:38,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598491) roll requested 2024-12-10T07:26:38,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598500 2024-12-10T07:26:38,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598500 2024-12-10T07:26:38,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598500) roll requested 2024-12-10T07:26:38,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598507 2024-12-10T07:26:38,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598507 2024-12-10T07:26:38,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598507) roll requested 2024-12-10T07:26:38,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598515 2024-12-10T07:26:38,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598515 2024-12-10T07:26:38,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598515) roll requested 2024-12-10T07:26:38,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598523 2024-12-10T07:26:38,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598523 2024-12-10T07:26:38,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598523) roll requested 2024-12-10T07:26:38,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598532 2024-12-10T07:26:38,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598532 2024-12-10T07:26:38,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598532) roll requested 2024-12-10T07:26:38,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598541 2024-12-10T07:26:38,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598541 2024-12-10T07:26:38,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598541) roll requested 2024-12-10T07:26:38,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598548 2024-12-10T07:26:38,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598548 2024-12-10T07:26:38,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598548) roll requested 2024-12-10T07:26:38,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598555 2024-12-10T07:26:38,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598555 2024-12-10T07:26:38,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598555) roll requested 2024-12-10T07:26:38,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598563 2024-12-10T07:26:38,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598563 2024-12-10T07:26:38,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598563) roll requested 2024-12-10T07:26:38,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598573 2024-12-10T07:26:38,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598573 2024-12-10T07:26:38,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598573) roll requested 2024-12-10T07:26:38,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598582 2024-12-10T07:26:38,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598582 2024-12-10T07:26:38,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598582) roll requested 2024-12-10T07:26:38,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598591 2024-12-10T07:26:38,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598591 2024-12-10T07:26:38,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598591) roll requested 2024-12-10T07:26:38,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598601 2024-12-10T07:26:38,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598601 2024-12-10T07:26:38,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598601) roll requested 2024-12-10T07:26:38,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598611 2024-12-10T07:26:38,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598611 2024-12-10T07:26:38,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598611) roll requested 2024-12-10T07:26:38,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598623 2024-12-10T07:26:38,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598623 2024-12-10T07:26:38,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598623) roll requested 2024-12-10T07:26:38,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598631 2024-12-10T07:26:38,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598631 2024-12-10T07:26:38,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598631) roll requested 2024-12-10T07:26:38,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598638 2024-12-10T07:26:38,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598638 2024-12-10T07:26:38,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598638) roll requested 2024-12-10T07:26:38,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598647 2024-12-10T07:26:38,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598647 2024-12-10T07:26:38,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598647) roll requested 2024-12-10T07:26:38,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598656 2024-12-10T07:26:38,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598656 2024-12-10T07:26:38,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598656) roll requested 2024-12-10T07:26:38,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598666 2024-12-10T07:26:38,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598666 2024-12-10T07:26:38,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598666) roll requested 2024-12-10T07:26:38,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598674 2024-12-10T07:26:38,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598674 2024-12-10T07:26:38,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598674) roll requested 2024-12-10T07:26:38,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598684 2024-12-10T07:26:38,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598684 2024-12-10T07:26:38,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598684) roll requested 2024-12-10T07:26:38,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598694 2024-12-10T07:26:38,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598694 2024-12-10T07:26:38,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598694) roll requested 2024-12-10T07:26:38,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598701 2024-12-10T07:26:38,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598701 2024-12-10T07:26:38,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598701) roll requested 2024-12-10T07:26:38,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598709 2024-12-10T07:26:38,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598709 2024-12-10T07:26:38,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598709) roll requested 2024-12-10T07:26:38,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598717 2024-12-10T07:26:38,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598717 2024-12-10T07:26:38,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598717) roll requested 2024-12-10T07:26:38,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598725 2024-12-10T07:26:38,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598725 2024-12-10T07:26:38,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598725) roll requested 2024-12-10T07:26:38,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598733 2024-12-10T07:26:38,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598733 2024-12-10T07:26:38,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598733) roll requested 2024-12-10T07:26:38,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598740 2024-12-10T07:26:38,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598740 2024-12-10T07:26:38,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598740) roll requested 2024-12-10T07:26:38,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598746 2024-12-10T07:26:38,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598746 2024-12-10T07:26:38,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598746) roll requested 2024-12-10T07:26:38,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598754 2024-12-10T07:26:38,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598754 2024-12-10T07:26:38,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598754) roll requested 2024-12-10T07:26:38,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598761 2024-12-10T07:26:38,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598761 2024-12-10T07:26:38,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598761) roll requested 2024-12-10T07:26:38,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598769 2024-12-10T07:26:38,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598769 2024-12-10T07:26:38,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598769) roll requested 2024-12-10T07:26:38,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598777 2024-12-10T07:26:38,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598777 2024-12-10T07:26:38,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598777) roll requested 2024-12-10T07:26:38,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598783 2024-12-10T07:26:38,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598783 2024-12-10T07:26:38,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598783) roll requested 2024-12-10T07:26:38,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598789 2024-12-10T07:26:38,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598789 2024-12-10T07:26:38,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598789) roll requested 2024-12-10T07:26:38,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598798 2024-12-10T07:26:38,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598798 2024-12-10T07:26:38,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598798) roll requested 2024-12-10T07:26:38,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598805 2024-12-10T07:26:38,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598805 2024-12-10T07:26:38,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598805) roll requested 2024-12-10T07:26:38,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598814 2024-12-10T07:26:38,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598814 2024-12-10T07:26:38,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598814) roll requested 2024-12-10T07:26:38,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598821 2024-12-10T07:26:38,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598821 2024-12-10T07:26:38,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598821) roll requested 2024-12-10T07:26:38,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598828 2024-12-10T07:26:38,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598828 2024-12-10T07:26:38,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598828) roll requested 2024-12-10T07:26:38,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598835 2024-12-10T07:26:38,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598835 2024-12-10T07:26:38,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598835) roll requested 2024-12-10T07:26:38,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598841 2024-12-10T07:26:38,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598841 2024-12-10T07:26:38,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598841) roll requested 2024-12-10T07:26:38,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598849 2024-12-10T07:26:38,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598849 2024-12-10T07:26:38,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598849) roll requested 2024-12-10T07:26:38,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598854 2024-12-10T07:26:38,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598854 2024-12-10T07:26:38,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598854) roll requested 2024-12-10T07:26:38,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598859 2024-12-10T07:26:38,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598859 2024-12-10T07:26:38,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598859) roll requested 2024-12-10T07:26:38,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598865 2024-12-10T07:26:38,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598865 2024-12-10T07:26:38,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598865) roll requested 2024-12-10T07:26:38,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598871 2024-12-10T07:26:38,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598871 2024-12-10T07:26:38,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598871) roll requested 2024-12-10T07:26:38,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598877 2024-12-10T07:26:38,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598877 2024-12-10T07:26:38,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598877) roll requested 2024-12-10T07:26:38,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598884 2024-12-10T07:26:38,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598884 2024-12-10T07:26:38,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598884) roll requested 2024-12-10T07:26:38,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598893 2024-12-10T07:26:38,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598893 2024-12-10T07:26:38,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598893) roll requested 2024-12-10T07:26:38,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598902 2024-12-10T07:26:38,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598902 2024-12-10T07:26:38,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598902) roll requested 2024-12-10T07:26:38,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598909 2024-12-10T07:26:38,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598909 2024-12-10T07:26:38,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598909) roll requested 2024-12-10T07:26:38,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598918 2024-12-10T07:26:38,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598918 2024-12-10T07:26:38,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598918) roll requested 2024-12-10T07:26:38,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598925 2024-12-10T07:26:38,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598925 2024-12-10T07:26:38,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598925) roll requested 2024-12-10T07:26:38,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598933 2024-12-10T07:26:38,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598933 2024-12-10T07:26:38,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598933) roll requested 2024-12-10T07:26:38,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598941 2024-12-10T07:26:38,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598941 2024-12-10T07:26:38,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598941) roll requested 2024-12-10T07:26:38,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598948 2024-12-10T07:26:38,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598948 2024-12-10T07:26:38,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598948) roll requested 2024-12-10T07:26:38,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598956 2024-12-10T07:26:38,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598956 2024-12-10T07:26:38,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598956) roll requested 2024-12-10T07:26:38,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598966 2024-12-10T07:26:38,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598966 2024-12-10T07:26:38,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598966) roll requested 2024-12-10T07:26:38,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598973 2024-12-10T07:26:38,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598973 2024-12-10T07:26:38,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598973) roll requested 2024-12-10T07:26:38,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598977 2024-12-10T07:26:38,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598977 2024-12-10T07:26:38,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598977) roll requested 2024-12-10T07:26:38,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598982 2024-12-10T07:26:38,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598982 2024-12-10T07:26:38,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598982) roll requested 2024-12-10T07:26:38,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598990 2024-12-10T07:26:38,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:38,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598990 2024-12-10T07:26:38,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:38,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:38,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:38,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:38,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598990) roll requested 2024-12-10T07:26:38,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815598998 2024-12-10T07:26:39,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598998 2024-12-10T07:26:39,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815598998) roll requested 2024-12-10T07:26:39,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599007 2024-12-10T07:26:39,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815598998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599007 2024-12-10T07:26:39,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599007) roll requested 2024-12-10T07:26:39,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599015 2024-12-10T07:26:39,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599015 2024-12-10T07:26:39,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599015) roll requested 2024-12-10T07:26:39,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599023 2024-12-10T07:26:39,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599023 2024-12-10T07:26:39,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599023) roll requested 2024-12-10T07:26:39,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599030 2024-12-10T07:26:39,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599030 2024-12-10T07:26:39,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599030) roll requested 2024-12-10T07:26:39,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599040 2024-12-10T07:26:39,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599040 2024-12-10T07:26:39,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599040) roll requested 2024-12-10T07:26:39,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599049 2024-12-10T07:26:39,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599049 2024-12-10T07:26:39,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599049) roll requested 2024-12-10T07:26:39,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599059 2024-12-10T07:26:39,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599059 2024-12-10T07:26:39,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599059) roll requested 2024-12-10T07:26:39,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599069 2024-12-10T07:26:39,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599069 2024-12-10T07:26:39,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599069) roll requested 2024-12-10T07:26:39,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599077 2024-12-10T07:26:39,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599077 2024-12-10T07:26:39,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599077) roll requested 2024-12-10T07:26:39,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599086 2024-12-10T07:26:39,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599086 2024-12-10T07:26:39,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599086) roll requested 2024-12-10T07:26:39,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599096 2024-12-10T07:26:39,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599096 2024-12-10T07:26:39,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599096) roll requested 2024-12-10T07:26:39,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599104 2024-12-10T07:26:39,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599104 2024-12-10T07:26:39,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599104) roll requested 2024-12-10T07:26:39,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599110 2024-12-10T07:26:39,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599110 2024-12-10T07:26:39,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599110) roll requested 2024-12-10T07:26:39,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599118 2024-12-10T07:26:39,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599118 2024-12-10T07:26:39,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599118) roll requested 2024-12-10T07:26:39,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599126 2024-12-10T07:26:39,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599126 2024-12-10T07:26:39,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599126) roll requested 2024-12-10T07:26:39,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599133 2024-12-10T07:26:39,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599133 2024-12-10T07:26:39,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599133) roll requested 2024-12-10T07:26:39,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599140 2024-12-10T07:26:39,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599140 2024-12-10T07:26:39,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599140) roll requested 2024-12-10T07:26:39,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599148 2024-12-10T07:26:39,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599148 2024-12-10T07:26:39,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599148) roll requested 2024-12-10T07:26:39,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599156 2024-12-10T07:26:39,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599156 2024-12-10T07:26:39,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599156) roll requested 2024-12-10T07:26:39,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599166 2024-12-10T07:26:39,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599166 2024-12-10T07:26:39,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599166) roll requested 2024-12-10T07:26:39,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599175 2024-12-10T07:26:39,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599175 2024-12-10T07:26:39,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599175) roll requested 2024-12-10T07:26:39,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599185 2024-12-10T07:26:39,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599185 2024-12-10T07:26:39,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599185) roll requested 2024-12-10T07:26:39,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599193 2024-12-10T07:26:39,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599193 2024-12-10T07:26:39,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599193) roll requested 2024-12-10T07:26:39,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599200 2024-12-10T07:26:39,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599200 2024-12-10T07:26:39,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599200) roll requested 2024-12-10T07:26:39,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599208 2024-12-10T07:26:39,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599208 2024-12-10T07:26:39,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599208) roll requested 2024-12-10T07:26:39,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599216 2024-12-10T07:26:39,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599216 2024-12-10T07:26:39,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599216) roll requested 2024-12-10T07:26:39,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599226 2024-12-10T07:26:39,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599226 2024-12-10T07:26:39,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599226) roll requested 2024-12-10T07:26:39,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599234 2024-12-10T07:26:39,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599234 2024-12-10T07:26:39,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599234) roll requested 2024-12-10T07:26:39,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599244 2024-12-10T07:26:39,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599244 2024-12-10T07:26:39,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599244) roll requested 2024-12-10T07:26:39,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599252 2024-12-10T07:26:39,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599252 2024-12-10T07:26:39,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599252) roll requested 2024-12-10T07:26:39,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599260 2024-12-10T07:26:39,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599260 2024-12-10T07:26:39,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599260) roll requested 2024-12-10T07:26:39,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599268 2024-12-10T07:26:39,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599268 2024-12-10T07:26:39,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599268) roll requested 2024-12-10T07:26:39,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599277 2024-12-10T07:26:39,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599277 2024-12-10T07:26:39,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599277) roll requested 2024-12-10T07:26:39,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599285 2024-12-10T07:26:39,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599285 2024-12-10T07:26:39,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599285) roll requested 2024-12-10T07:26:39,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599292 2024-12-10T07:26:39,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599292 2024-12-10T07:26:39,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599292) roll requested 2024-12-10T07:26:39,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599298 2024-12-10T07:26:39,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599298 2024-12-10T07:26:39,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599298) roll requested 2024-12-10T07:26:39,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599304 2024-12-10T07:26:39,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599304 2024-12-10T07:26:39,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599304) roll requested 2024-12-10T07:26:39,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599311 2024-12-10T07:26:39,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599311 2024-12-10T07:26:39,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599311) roll requested 2024-12-10T07:26:39,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599317 2024-12-10T07:26:39,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599317 2024-12-10T07:26:39,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599317) roll requested 2024-12-10T07:26:39,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599323 2024-12-10T07:26:39,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599323 2024-12-10T07:26:39,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599323) roll requested 2024-12-10T07:26:39,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599329 2024-12-10T07:26:39,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599329 2024-12-10T07:26:39,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599329) roll requested 2024-12-10T07:26:39,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599337 2024-12-10T07:26:39,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599337 2024-12-10T07:26:39,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599337) roll requested 2024-12-10T07:26:39,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599345 2024-12-10T07:26:39,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599345 2024-12-10T07:26:39,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599345) roll requested 2024-12-10T07:26:39,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599353 2024-12-10T07:26:39,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599353 2024-12-10T07:26:39,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599353) roll requested 2024-12-10T07:26:39,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599361 2024-12-10T07:26:39,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599361 2024-12-10T07:26:39,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599361) roll requested 2024-12-10T07:26:39,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599370 2024-12-10T07:26:39,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599370 2024-12-10T07:26:39,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599370) roll requested 2024-12-10T07:26:39,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599379 2024-12-10T07:26:39,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599379 2024-12-10T07:26:39,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599379) roll requested 2024-12-10T07:26:39,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599389 2024-12-10T07:26:39,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599389 2024-12-10T07:26:39,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599389) roll requested 2024-12-10T07:26:39,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599398 2024-12-10T07:26:39,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599398 2024-12-10T07:26:39,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599398) roll requested 2024-12-10T07:26:39,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599407 2024-12-10T07:26:39,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599407 2024-12-10T07:26:39,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599407) roll requested 2024-12-10T07:26:39,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599414 2024-12-10T07:26:39,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599414 2024-12-10T07:26:39,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599414) roll requested 2024-12-10T07:26:39,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599421 2024-12-10T07:26:39,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599421 2024-12-10T07:26:39,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599421) roll requested 2024-12-10T07:26:39,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599429 2024-12-10T07:26:39,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599429 2024-12-10T07:26:39,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599429) roll requested 2024-12-10T07:26:39,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599436 2024-12-10T07:26:39,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599436 2024-12-10T07:26:39,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599436) roll requested 2024-12-10T07:26:39,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599444 2024-12-10T07:26:39,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599444 2024-12-10T07:26:39,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599444) roll requested 2024-12-10T07:26:39,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599453 2024-12-10T07:26:39,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599453 2024-12-10T07:26:39,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599453) roll requested 2024-12-10T07:26:39,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599464 2024-12-10T07:26:39,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599464 2024-12-10T07:26:39,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599464) roll requested 2024-12-10T07:26:39,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599472 2024-12-10T07:26:39,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599472 2024-12-10T07:26:39,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599472) roll requested 2024-12-10T07:26:39,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599479 2024-12-10T07:26:39,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599479 2024-12-10T07:26:39,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599479) roll requested 2024-12-10T07:26:39,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599488 2024-12-10T07:26:39,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599488 2024-12-10T07:26:39,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599488) roll requested 2024-12-10T07:26:39,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599497 2024-12-10T07:26:39,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599497 2024-12-10T07:26:39,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599497) roll requested 2024-12-10T07:26:39,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599508 2024-12-10T07:26:39,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599508 2024-12-10T07:26:39,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599508) roll requested 2024-12-10T07:26:39,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599518 2024-12-10T07:26:39,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599518 2024-12-10T07:26:39,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599518) roll requested 2024-12-10T07:26:39,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599526 2024-12-10T07:26:39,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599526 2024-12-10T07:26:39,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599526) roll requested 2024-12-10T07:26:39,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599535 2024-12-10T07:26:39,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599535 2024-12-10T07:26:39,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599535) roll requested 2024-12-10T07:26:39,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599543 2024-12-10T07:26:39,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599543 2024-12-10T07:26:39,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599543) roll requested 2024-12-10T07:26:39,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599551 2024-12-10T07:26:39,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599551 2024-12-10T07:26:39,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599551) roll requested 2024-12-10T07:26:39,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599557 2024-12-10T07:26:39,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599557 2024-12-10T07:26:39,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599557) roll requested 2024-12-10T07:26:39,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599567 2024-12-10T07:26:39,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599567 2024-12-10T07:26:39,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599567) roll requested 2024-12-10T07:26:39,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599575 2024-12-10T07:26:39,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599575 2024-12-10T07:26:39,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599575) roll requested 2024-12-10T07:26:39,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599581 2024-12-10T07:26:39,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599581 2024-12-10T07:26:39,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599581) roll requested 2024-12-10T07:26:39,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599588 2024-12-10T07:26:39,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599588 2024-12-10T07:26:39,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599588) roll requested 2024-12-10T07:26:39,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599596 2024-12-10T07:26:39,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599596 2024-12-10T07:26:39,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599596) roll requested 2024-12-10T07:26:39,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599604 2024-12-10T07:26:39,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599604 2024-12-10T07:26:39,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599604) roll requested 2024-12-10T07:26:39,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599611 2024-12-10T07:26:39,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599611 2024-12-10T07:26:39,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599611) roll requested 2024-12-10T07:26:39,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599620 2024-12-10T07:26:39,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599620 2024-12-10T07:26:39,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599620) roll requested 2024-12-10T07:26:39,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599629 2024-12-10T07:26:39,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599629 2024-12-10T07:26:39,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599629) roll requested 2024-12-10T07:26:39,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599640 2024-12-10T07:26:39,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599640 2024-12-10T07:26:39,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599640) roll requested 2024-12-10T07:26:39,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599648 2024-12-10T07:26:39,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599648 2024-12-10T07:26:39,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599648) roll requested 2024-12-10T07:26:39,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599657 2024-12-10T07:26:39,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599657 2024-12-10T07:26:39,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599657) roll requested 2024-12-10T07:26:39,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599667 2024-12-10T07:26:39,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599667 2024-12-10T07:26:39,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599667) roll requested 2024-12-10T07:26:39,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599676 2024-12-10T07:26:39,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599676 2024-12-10T07:26:39,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599676) roll requested 2024-12-10T07:26:39,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599685 2024-12-10T07:26:39,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599685 2024-12-10T07:26:39,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599685) roll requested 2024-12-10T07:26:39,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599694 2024-12-10T07:26:39,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599694 2024-12-10T07:26:39,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599694) roll requested 2024-12-10T07:26:39,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599701 2024-12-10T07:26:39,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599701 2024-12-10T07:26:39,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599701) roll requested 2024-12-10T07:26:39,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599709 2024-12-10T07:26:39,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599709 2024-12-10T07:26:39,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599709) roll requested 2024-12-10T07:26:39,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599718 2024-12-10T07:26:39,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599718 2024-12-10T07:26:39,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599718) roll requested 2024-12-10T07:26:39,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599725 2024-12-10T07:26:39,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599725 2024-12-10T07:26:39,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599725) roll requested 2024-12-10T07:26:39,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599733 2024-12-10T07:26:39,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599733 2024-12-10T07:26:39,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599733) roll requested 2024-12-10T07:26:39,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599740 2024-12-10T07:26:39,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599740 2024-12-10T07:26:39,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599740) roll requested 2024-12-10T07:26:39,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599748 2024-12-10T07:26:39,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599748 2024-12-10T07:26:39,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599748) roll requested 2024-12-10T07:26:39,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599758 2024-12-10T07:26:39,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599758 2024-12-10T07:26:39,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599758) roll requested 2024-12-10T07:26:39,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599769 2024-12-10T07:26:39,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599769 2024-12-10T07:26:39,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599769) roll requested 2024-12-10T07:26:39,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599778 2024-12-10T07:26:39,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599778 2024-12-10T07:26:39,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599778) roll requested 2024-12-10T07:26:39,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599786 2024-12-10T07:26:39,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599786 2024-12-10T07:26:39,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599786) roll requested 2024-12-10T07:26:39,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599793 2024-12-10T07:26:39,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599793 2024-12-10T07:26:39,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599793) roll requested 2024-12-10T07:26:39,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599801 2024-12-10T07:26:39,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599801 2024-12-10T07:26:39,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599801) roll requested 2024-12-10T07:26:39,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599809 2024-12-10T07:26:39,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599809 2024-12-10T07:26:39,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599809) roll requested 2024-12-10T07:26:39,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599817 2024-12-10T07:26:39,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599817 2024-12-10T07:26:39,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599817) roll requested 2024-12-10T07:26:39,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599824 2024-12-10T07:26:39,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599824 2024-12-10T07:26:39,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599824) roll requested 2024-12-10T07:26:39,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599833 2024-12-10T07:26:39,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599833 2024-12-10T07:26:39,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599833) roll requested 2024-12-10T07:26:39,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599842 2024-12-10T07:26:39,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599842 2024-12-10T07:26:39,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599842) roll requested 2024-12-10T07:26:39,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599852 2024-12-10T07:26:39,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599852 2024-12-10T07:26:39,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599852) roll requested 2024-12-10T07:26:39,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599862 2024-12-10T07:26:39,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599862 2024-12-10T07:26:39,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599862) roll requested 2024-12-10T07:26:39,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599870 2024-12-10T07:26:39,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599870 2024-12-10T07:26:39,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599870) roll requested 2024-12-10T07:26:39,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599878 2024-12-10T07:26:39,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599878 2024-12-10T07:26:39,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599878) roll requested 2024-12-10T07:26:39,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599886 2024-12-10T07:26:39,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599886 2024-12-10T07:26:39,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599886) roll requested 2024-12-10T07:26:39,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599894 2024-12-10T07:26:39,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599894 2024-12-10T07:26:39,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599894) roll requested 2024-12-10T07:26:39,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599899 2024-12-10T07:26:39,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599899 2024-12-10T07:26:39,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599899) roll requested 2024-12-10T07:26:39,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599904 2024-12-10T07:26:39,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599904 2024-12-10T07:26:39,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599904) roll requested 2024-12-10T07:26:39,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599911 2024-12-10T07:26:39,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599911 2024-12-10T07:26:39,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599911) roll requested 2024-12-10T07:26:39,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599917 2024-12-10T07:26:39,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599917 2024-12-10T07:26:39,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599917) roll requested 2024-12-10T07:26:39,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599925 2024-12-10T07:26:39,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599925 2024-12-10T07:26:39,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599925) roll requested 2024-12-10T07:26:39,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599932 2024-12-10T07:26:39,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599932 2024-12-10T07:26:39,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599932) roll requested 2024-12-10T07:26:39,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599939 2024-12-10T07:26:39,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599939 2024-12-10T07:26:39,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599939) roll requested 2024-12-10T07:26:39,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599946 2024-12-10T07:26:39,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599946 2024-12-10T07:26:39,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599946) roll requested 2024-12-10T07:26:39,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599954 2024-12-10T07:26:39,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599954 2024-12-10T07:26:39,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599954) roll requested 2024-12-10T07:26:39,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599963 2024-12-10T07:26:39,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599963 2024-12-10T07:26:39,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599963) roll requested 2024-12-10T07:26:39,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599974 2024-12-10T07:26:39,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599963 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599974 2024-12-10T07:26:39,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599974) roll requested 2024-12-10T07:26:39,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599982 2024-12-10T07:26:39,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599982 2024-12-10T07:26:39,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599982) roll requested 2024-12-10T07:26:39,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599988 2024-12-10T07:26:39,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:39,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599988 2024-12-10T07:26:39,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:39,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:39,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:39,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:39,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599988) roll requested 2024-12-10T07:26:39,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815599995 2024-12-10T07:26:40,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599995 2024-12-10T07:26:40,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815599995) roll requested 2024-12-10T07:26:40,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600002 2024-12-10T07:26:40,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815599995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600002 2024-12-10T07:26:40,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600002) roll requested 2024-12-10T07:26:40,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600011 2024-12-10T07:26:40,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600011 2024-12-10T07:26:40,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600011) roll requested 2024-12-10T07:26:40,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600022 2024-12-10T07:26:40,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600022 2024-12-10T07:26:40,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600022) roll requested 2024-12-10T07:26:40,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600032 2024-12-10T07:26:40,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600032 2024-12-10T07:26:40,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600032) roll requested 2024-12-10T07:26:40,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600037 2024-12-10T07:26:40,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600037 2024-12-10T07:26:40,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600037) roll requested 2024-12-10T07:26:40,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600045 2024-12-10T07:26:40,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600045 2024-12-10T07:26:40,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600045) roll requested 2024-12-10T07:26:40,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600052 2024-12-10T07:26:40,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600052 2024-12-10T07:26:40,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600052) roll requested 2024-12-10T07:26:40,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600058 2024-12-10T07:26:40,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600058 2024-12-10T07:26:40,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600058) roll requested 2024-12-10T07:26:40,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600064 2024-12-10T07:26:40,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600064 2024-12-10T07:26:40,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600064) roll requested 2024-12-10T07:26:40,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600073 2024-12-10T07:26:40,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600073 2024-12-10T07:26:40,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600073) roll requested 2024-12-10T07:26:40,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600079 2024-12-10T07:26:40,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600079 2024-12-10T07:26:40,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600079) roll requested 2024-12-10T07:26:40,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600085 2024-12-10T07:26:40,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600085 2024-12-10T07:26:40,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600085) roll requested 2024-12-10T07:26:40,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600090 2024-12-10T07:26:40,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600090 2024-12-10T07:26:40,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600090) roll requested 2024-12-10T07:26:40,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600157 2024-12-10T07:26:40,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600157 2024-12-10T07:26:40,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600157) roll requested 2024-12-10T07:26:40,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600166 2024-12-10T07:26:40,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600166 2024-12-10T07:26:40,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600166) roll requested 2024-12-10T07:26:40,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600173 2024-12-10T07:26:40,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600173 2024-12-10T07:26:40,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600173) roll requested 2024-12-10T07:26:40,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600181 2024-12-10T07:26:40,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600181 2024-12-10T07:26:40,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600181) roll requested 2024-12-10T07:26:40,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600188 2024-12-10T07:26:40,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600188 2024-12-10T07:26:40,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600188) roll requested 2024-12-10T07:26:40,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600196 2024-12-10T07:26:40,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600196 2024-12-10T07:26:40,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600196) roll requested 2024-12-10T07:26:40,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600205 2024-12-10T07:26:40,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600205 2024-12-10T07:26:40,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600205) roll requested 2024-12-10T07:26:40,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600214 2024-12-10T07:26:40,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600214 2024-12-10T07:26:40,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600214) roll requested 2024-12-10T07:26:40,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600222 2024-12-10T07:26:40,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600222 2024-12-10T07:26:40,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600222) roll requested 2024-12-10T07:26:40,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600230 2024-12-10T07:26:40,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600230 2024-12-10T07:26:40,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600230) roll requested 2024-12-10T07:26:40,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600239 2024-12-10T07:26:40,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600239 2024-12-10T07:26:40,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600239) roll requested 2024-12-10T07:26:40,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600248 2024-12-10T07:26:40,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600248 2024-12-10T07:26:40,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600248) roll requested 2024-12-10T07:26:40,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600256 2024-12-10T07:26:40,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600256 2024-12-10T07:26:40,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600256) roll requested 2024-12-10T07:26:40,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600266 2024-12-10T07:26:40,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600266 2024-12-10T07:26:40,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600266) roll requested 2024-12-10T07:26:40,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600275 2024-12-10T07:26:40,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600275 2024-12-10T07:26:40,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600275) roll requested 2024-12-10T07:26:40,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600282 2024-12-10T07:26:40,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600282 2024-12-10T07:26:40,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600282) roll requested 2024-12-10T07:26:40,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600289 2024-12-10T07:26:40,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600289 2024-12-10T07:26:40,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600289) roll requested 2024-12-10T07:26:40,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600297 2024-12-10T07:26:40,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600297 2024-12-10T07:26:40,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600297) roll requested 2024-12-10T07:26:40,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600304 2024-12-10T07:26:40,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600304 2024-12-10T07:26:40,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600304) roll requested 2024-12-10T07:26:40,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600314 2024-12-10T07:26:40,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600314 2024-12-10T07:26:40,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600314) roll requested 2024-12-10T07:26:40,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600324 2024-12-10T07:26:40,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600324 2024-12-10T07:26:40,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600324) roll requested 2024-12-10T07:26:40,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600332 2024-12-10T07:26:40,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600332 2024-12-10T07:26:40,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600332) roll requested 2024-12-10T07:26:40,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600341 2024-12-10T07:26:40,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600341 2024-12-10T07:26:40,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600341) roll requested 2024-12-10T07:26:40,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600350 2024-12-10T07:26:40,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600350 2024-12-10T07:26:40,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600350) roll requested 2024-12-10T07:26:40,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600356 2024-12-10T07:26:40,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600356 2024-12-10T07:26:40,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600356) roll requested 2024-12-10T07:26:40,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600365 2024-12-10T07:26:40,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600365 2024-12-10T07:26:40,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600365) roll requested 2024-12-10T07:26:40,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600373 2024-12-10T07:26:40,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600373 2024-12-10T07:26:40,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600373) roll requested 2024-12-10T07:26:40,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600380 2024-12-10T07:26:40,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600380 2024-12-10T07:26:40,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600380) roll requested 2024-12-10T07:26:40,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600390 2024-12-10T07:26:40,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600390 2024-12-10T07:26:40,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600390) roll requested 2024-12-10T07:26:40,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600398 2024-12-10T07:26:40,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600398 2024-12-10T07:26:40,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600398) roll requested 2024-12-10T07:26:40,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600404 2024-12-10T07:26:40,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600404 2024-12-10T07:26:40,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600404) roll requested 2024-12-10T07:26:40,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600411 2024-12-10T07:26:40,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600411 2024-12-10T07:26:40,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600411) roll requested 2024-12-10T07:26:40,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600419 2024-12-10T07:26:40,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600419 2024-12-10T07:26:40,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600419) roll requested 2024-12-10T07:26:40,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600428 2024-12-10T07:26:40,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600428 2024-12-10T07:26:40,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600428) roll requested 2024-12-10T07:26:40,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600438 2024-12-10T07:26:40,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600438 2024-12-10T07:26:40,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600438) roll requested 2024-12-10T07:26:40,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600447 2024-12-10T07:26:40,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600447 2024-12-10T07:26:40,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600447) roll requested 2024-12-10T07:26:40,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600456 2024-12-10T07:26:40,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600456 2024-12-10T07:26:40,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600456) roll requested 2024-12-10T07:26:40,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600464 2024-12-10T07:26:40,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600464 2024-12-10T07:26:40,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600464) roll requested 2024-12-10T07:26:40,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600472 2024-12-10T07:26:40,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600472 2024-12-10T07:26:40,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600472) roll requested 2024-12-10T07:26:40,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600481 2024-12-10T07:26:40,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600481 2024-12-10T07:26:40,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600481) roll requested 2024-12-10T07:26:40,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600487 2024-12-10T07:26:40,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600487 2024-12-10T07:26:40,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600487) roll requested 2024-12-10T07:26:40,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600494 2024-12-10T07:26:40,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600494 2024-12-10T07:26:40,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600494) roll requested 2024-12-10T07:26:40,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600501 2024-12-10T07:26:40,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600501 2024-12-10T07:26:40,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600501) roll requested 2024-12-10T07:26:40,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600510 2024-12-10T07:26:40,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600510 2024-12-10T07:26:40,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600510) roll requested 2024-12-10T07:26:40,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600519 2024-12-10T07:26:40,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600519 2024-12-10T07:26:40,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600519) roll requested 2024-12-10T07:26:40,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600529 2024-12-10T07:26:40,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600529 2024-12-10T07:26:40,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600529) roll requested 2024-12-10T07:26:40,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600537 2024-12-10T07:26:40,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600537 2024-12-10T07:26:40,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600537) roll requested 2024-12-10T07:26:40,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600547 2024-12-10T07:26:40,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600547 2024-12-10T07:26:40,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600547) roll requested 2024-12-10T07:26:40,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600557 2024-12-10T07:26:40,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600557 2024-12-10T07:26:40,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600557) roll requested 2024-12-10T07:26:40,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600563 2024-12-10T07:26:40,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600563 2024-12-10T07:26:40,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600563) roll requested 2024-12-10T07:26:40,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600570 2024-12-10T07:26:40,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600570 2024-12-10T07:26:40,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600570) roll requested 2024-12-10T07:26:40,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600577 2024-12-10T07:26:40,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600577 2024-12-10T07:26:40,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600577) roll requested 2024-12-10T07:26:40,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600582 2024-12-10T07:26:40,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600582 2024-12-10T07:26:40,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600582) roll requested 2024-12-10T07:26:40,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600588 2024-12-10T07:26:40,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600588 2024-12-10T07:26:40,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600588) roll requested 2024-12-10T07:26:40,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600597 2024-12-10T07:26:40,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600597 2024-12-10T07:26:40,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600597) roll requested 2024-12-10T07:26:40,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600603 2024-12-10T07:26:40,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600603 2024-12-10T07:26:40,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600603) roll requested 2024-12-10T07:26:40,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600609 2024-12-10T07:26:40,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600609 2024-12-10T07:26:40,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600609) roll requested 2024-12-10T07:26:40,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600614 2024-12-10T07:26:40,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600614 2024-12-10T07:26:40,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600614) roll requested 2024-12-10T07:26:40,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600620 2024-12-10T07:26:40,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600620 2024-12-10T07:26:40,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600620) roll requested 2024-12-10T07:26:40,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600626 2024-12-10T07:26:40,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600626 2024-12-10T07:26:40,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600626) roll requested 2024-12-10T07:26:40,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600633 2024-12-10T07:26:40,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600633 2024-12-10T07:26:40,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600633) roll requested 2024-12-10T07:26:40,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600640 2024-12-10T07:26:40,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600640 2024-12-10T07:26:40,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600640) roll requested 2024-12-10T07:26:40,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600649 2024-12-10T07:26:40,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600649 2024-12-10T07:26:40,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600649) roll requested 2024-12-10T07:26:40,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600659 2024-12-10T07:26:40,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600659 2024-12-10T07:26:40,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600659) roll requested 2024-12-10T07:26:40,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600668 2024-12-10T07:26:40,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600668 2024-12-10T07:26:40,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600668) roll requested 2024-12-10T07:26:40,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600676 2024-12-10T07:26:40,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600676 2024-12-10T07:26:40,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600676) roll requested 2024-12-10T07:26:40,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600685 2024-12-10T07:26:40,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600685 2024-12-10T07:26:40,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600685) roll requested 2024-12-10T07:26:40,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600691 2024-12-10T07:26:40,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600691 2024-12-10T07:26:40,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600691) roll requested 2024-12-10T07:26:40,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600701 2024-12-10T07:26:40,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600701 2024-12-10T07:26:40,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600701) roll requested 2024-12-10T07:26:40,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600706 2024-12-10T07:26:40,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600706 2024-12-10T07:26:40,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600706) roll requested 2024-12-10T07:26:40,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600717 2024-12-10T07:26:40,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600717 2024-12-10T07:26:40,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600717) roll requested 2024-12-10T07:26:40,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600723 2024-12-10T07:26:40,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600723 2024-12-10T07:26:40,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600723) roll requested 2024-12-10T07:26:40,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600728 2024-12-10T07:26:40,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600728 2024-12-10T07:26:40,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600728) roll requested 2024-12-10T07:26:40,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600734 2024-12-10T07:26:40,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600734 2024-12-10T07:26:40,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600734) roll requested 2024-12-10T07:26:40,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600741 2024-12-10T07:26:40,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600741 2024-12-10T07:26:40,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600741) roll requested 2024-12-10T07:26:40,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600749 2024-12-10T07:26:40,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600749 2024-12-10T07:26:40,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600749) roll requested 2024-12-10T07:26:40,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600756 2024-12-10T07:26:40,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600756 2024-12-10T07:26:40,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600756) roll requested 2024-12-10T07:26:40,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600763 2024-12-10T07:26:40,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600763 2024-12-10T07:26:40,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600763) roll requested 2024-12-10T07:26:40,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600773 2024-12-10T07:26:40,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600773 2024-12-10T07:26:40,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600773) roll requested 2024-12-10T07:26:40,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600782 2024-12-10T07:26:40,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600782 2024-12-10T07:26:40,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600782) roll requested 2024-12-10T07:26:40,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600792 2024-12-10T07:26:40,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600792 2024-12-10T07:26:40,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600792) roll requested 2024-12-10T07:26:40,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600802 2024-12-10T07:26:40,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600802 2024-12-10T07:26:40,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600802) roll requested 2024-12-10T07:26:40,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600813 2024-12-10T07:26:40,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600813 2024-12-10T07:26:40,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600813) roll requested 2024-12-10T07:26:40,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600822 2024-12-10T07:26:40,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600822 2024-12-10T07:26:40,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600822) roll requested 2024-12-10T07:26:40,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600831 2024-12-10T07:26:40,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600831 2024-12-10T07:26:40,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600831) roll requested 2024-12-10T07:26:40,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600839 2024-12-10T07:26:40,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600839 2024-12-10T07:26:40,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600839) roll requested 2024-12-10T07:26:40,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600847 2024-12-10T07:26:40,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600847 2024-12-10T07:26:40,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600847) roll requested 2024-12-10T07:26:40,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600857 2024-12-10T07:26:40,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600857 2024-12-10T07:26:40,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600857) roll requested 2024-12-10T07:26:40,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600865 2024-12-10T07:26:40,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600865 2024-12-10T07:26:40,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600865) roll requested 2024-12-10T07:26:40,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600873 2024-12-10T07:26:40,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600873 2024-12-10T07:26:40,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600873) roll requested 2024-12-10T07:26:40,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600882 2024-12-10T07:26:40,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600882 2024-12-10T07:26:40,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600882) roll requested 2024-12-10T07:26:40,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600889 2024-12-10T07:26:40,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600889 2024-12-10T07:26:40,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600889) roll requested 2024-12-10T07:26:40,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600897 2024-12-10T07:26:40,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600897 2024-12-10T07:26:40,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600897) roll requested 2024-12-10T07:26:40,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600906 2024-12-10T07:26:40,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600906 2024-12-10T07:26:40,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600906) roll requested 2024-12-10T07:26:40,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600915 2024-12-10T07:26:40,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600915 2024-12-10T07:26:40,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600915) roll requested 2024-12-10T07:26:40,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600922 2024-12-10T07:26:40,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600922 2024-12-10T07:26:40,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600922) roll requested 2024-12-10T07:26:40,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600930 2024-12-10T07:26:40,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600930 2024-12-10T07:26:40,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600930) roll requested 2024-12-10T07:26:40,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600939 2024-12-10T07:26:40,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600939 2024-12-10T07:26:40,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600939) roll requested 2024-12-10T07:26:40,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600947 2024-12-10T07:26:40,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600947 2024-12-10T07:26:40,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600947) roll requested 2024-12-10T07:26:40,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600956 2024-12-10T07:26:40,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600956 2024-12-10T07:26:40,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600956) roll requested 2024-12-10T07:26:40,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600965 2024-12-10T07:26:40,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600965 2024-12-10T07:26:40,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600965) roll requested 2024-12-10T07:26:40,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600977 2024-12-10T07:26:40,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600977 2024-12-10T07:26:40,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600977) roll requested 2024-12-10T07:26:40,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600982 2024-12-10T07:26:40,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600982 2024-12-10T07:26:40,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600982) roll requested 2024-12-10T07:26:40,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600989 2024-12-10T07:26:40,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:40,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600989 2024-12-10T07:26:40,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:40,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:40,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:40,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:40,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600989) roll requested 2024-12-10T07:26:40,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815600995 2024-12-10T07:26:41,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600995 2024-12-10T07:26:41,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815600995) roll requested 2024-12-10T07:26:41,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601005 2024-12-10T07:26:41,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815600995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601005 2024-12-10T07:26:41,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601005) roll requested 2024-12-10T07:26:41,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601011 2024-12-10T07:26:41,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601011 2024-12-10T07:26:41,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601011) roll requested 2024-12-10T07:26:41,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601017 2024-12-10T07:26:41,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601017 2024-12-10T07:26:41,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601017) roll requested 2024-12-10T07:26:41,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601029 2024-12-10T07:26:41,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601029 2024-12-10T07:26:41,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601029) roll requested 2024-12-10T07:26:41,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601035 2024-12-10T07:26:41,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601035 2024-12-10T07:26:41,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601035) roll requested 2024-12-10T07:26:41,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601046 2024-12-10T07:26:41,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601046 2024-12-10T07:26:41,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601046) roll requested 2024-12-10T07:26:41,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601056 2024-12-10T07:26:41,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601056 2024-12-10T07:26:41,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601056) roll requested 2024-12-10T07:26:41,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601064 2024-12-10T07:26:41,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601064 2024-12-10T07:26:41,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601064) roll requested 2024-12-10T07:26:41,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601072 2024-12-10T07:26:41,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601072 2024-12-10T07:26:41,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601072) roll requested 2024-12-10T07:26:41,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601077 2024-12-10T07:26:41,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601077 2024-12-10T07:26:41,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601077) roll requested 2024-12-10T07:26:41,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601084 2024-12-10T07:26:41,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601084 2024-12-10T07:26:41,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601084) roll requested 2024-12-10T07:26:41,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601093 2024-12-10T07:26:41,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601093 2024-12-10T07:26:41,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601093) roll requested 2024-12-10T07:26:41,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601104 2024-12-10T07:26:41,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601104 2024-12-10T07:26:41,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601104) roll requested 2024-12-10T07:26:41,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601109 2024-12-10T07:26:41,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601109 2024-12-10T07:26:41,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601109) roll requested 2024-12-10T07:26:41,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601121 2024-12-10T07:26:41,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601121 2024-12-10T07:26:41,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601121) roll requested 2024-12-10T07:26:41,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601129 2024-12-10T07:26:41,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601129 2024-12-10T07:26:41,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601129) roll requested 2024-12-10T07:26:41,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601135 2024-12-10T07:26:41,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601135 2024-12-10T07:26:41,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601135) roll requested 2024-12-10T07:26:41,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601140 2024-12-10T07:26:41,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601140 2024-12-10T07:26:41,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601140) roll requested 2024-12-10T07:26:41,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601145 2024-12-10T07:26:41,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601145 2024-12-10T07:26:41,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601145) roll requested 2024-12-10T07:26:41,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601151 2024-12-10T07:26:41,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601151 2024-12-10T07:26:41,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601151) roll requested 2024-12-10T07:26:41,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601157 2024-12-10T07:26:41,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601157 2024-12-10T07:26:41,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601157) roll requested 2024-12-10T07:26:41,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601165 2024-12-10T07:26:41,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601165 2024-12-10T07:26:41,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601165) roll requested 2024-12-10T07:26:41,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601173 2024-12-10T07:26:41,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601173 2024-12-10T07:26:41,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601173) roll requested 2024-12-10T07:26:41,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601180 2024-12-10T07:26:41,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601180 2024-12-10T07:26:41,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601180) roll requested 2024-12-10T07:26:41,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601189 2024-12-10T07:26:41,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601189 2024-12-10T07:26:41,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601189) roll requested 2024-12-10T07:26:41,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601198 2024-12-10T07:26:41,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601198 2024-12-10T07:26:41,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601198) roll requested 2024-12-10T07:26:41,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601209 2024-12-10T07:26:41,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601209 2024-12-10T07:26:41,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601209) roll requested 2024-12-10T07:26:41,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601218 2024-12-10T07:26:41,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601218 2024-12-10T07:26:41,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601218) roll requested 2024-12-10T07:26:41,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601227 2024-12-10T07:26:41,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601227 2024-12-10T07:26:41,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601227) roll requested 2024-12-10T07:26:41,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601236 2024-12-10T07:26:41,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601236 2024-12-10T07:26:41,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601236) roll requested 2024-12-10T07:26:41,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601243 2024-12-10T07:26:41,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601243 2024-12-10T07:26:41,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601243) roll requested 2024-12-10T07:26:41,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601250 2024-12-10T07:26:41,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601250 2024-12-10T07:26:41,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601250) roll requested 2024-12-10T07:26:41,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601259 2024-12-10T07:26:41,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601259 2024-12-10T07:26:41,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601259) roll requested 2024-12-10T07:26:41,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601269 2024-12-10T07:26:41,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601269 2024-12-10T07:26:41,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601269) roll requested 2024-12-10T07:26:41,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601277 2024-12-10T07:26:41,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601277 2024-12-10T07:26:41,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601277) roll requested 2024-12-10T07:26:41,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601285 2024-12-10T07:26:41,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601285 2024-12-10T07:26:41,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601285) roll requested 2024-12-10T07:26:41,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601292 2024-12-10T07:26:41,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601292 2024-12-10T07:26:41,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601292) roll requested 2024-12-10T07:26:41,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601299 2024-12-10T07:26:41,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601299 2024-12-10T07:26:41,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601299) roll requested 2024-12-10T07:26:41,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601305 2024-12-10T07:26:41,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601305 2024-12-10T07:26:41,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601305) roll requested 2024-12-10T07:26:41,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601312 2024-12-10T07:26:41,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601312 2024-12-10T07:26:41,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601312) roll requested 2024-12-10T07:26:41,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601320 2024-12-10T07:26:41,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601320 2024-12-10T07:26:41,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601320) roll requested 2024-12-10T07:26:41,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601328 2024-12-10T07:26:41,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601328 2024-12-10T07:26:41,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601328) roll requested 2024-12-10T07:26:41,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601336 2024-12-10T07:26:41,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601336 2024-12-10T07:26:41,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601336) roll requested 2024-12-10T07:26:41,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601345 2024-12-10T07:26:41,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601345 2024-12-10T07:26:41,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601345) roll requested 2024-12-10T07:26:41,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601354 2024-12-10T07:26:41,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601354 2024-12-10T07:26:41,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601354) roll requested 2024-12-10T07:26:41,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601363 2024-12-10T07:26:41,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601363 2024-12-10T07:26:41,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601363) roll requested 2024-12-10T07:26:41,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601374 2024-12-10T07:26:41,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601374 2024-12-10T07:26:41,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601374) roll requested 2024-12-10T07:26:41,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601381 2024-12-10T07:26:41,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601381 2024-12-10T07:26:41,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601381) roll requested 2024-12-10T07:26:41,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601387 2024-12-10T07:26:41,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601387 2024-12-10T07:26:41,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601387) roll requested 2024-12-10T07:26:41,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601393 2024-12-10T07:26:41,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601393 2024-12-10T07:26:41,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601393) roll requested 2024-12-10T07:26:41,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601400 2024-12-10T07:26:41,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601400 2024-12-10T07:26:41,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601400) roll requested 2024-12-10T07:26:41,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601408 2024-12-10T07:26:41,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601408 2024-12-10T07:26:41,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601408) roll requested 2024-12-10T07:26:41,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601415 2024-12-10T07:26:41,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601415 2024-12-10T07:26:41,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601415) roll requested 2024-12-10T07:26:41,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601425 2024-12-10T07:26:41,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601425 2024-12-10T07:26:41,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601425) roll requested 2024-12-10T07:26:41,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601433 2024-12-10T07:26:41,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601433 2024-12-10T07:26:41,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601433) roll requested 2024-12-10T07:26:41,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601438 2024-12-10T07:26:41,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601438 2024-12-10T07:26:41,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601438) roll requested 2024-12-10T07:26:41,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601444 2024-12-10T07:26:41,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601444 2024-12-10T07:26:41,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601444) roll requested 2024-12-10T07:26:41,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601451 2024-12-10T07:26:41,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601451 2024-12-10T07:26:41,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601451) roll requested 2024-12-10T07:26:41,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601459 2024-12-10T07:26:41,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601459 2024-12-10T07:26:41,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601459) roll requested 2024-12-10T07:26:41,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601467 2024-12-10T07:26:41,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601467 2024-12-10T07:26:41,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601467) roll requested 2024-12-10T07:26:41,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601475 2024-12-10T07:26:41,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601475 2024-12-10T07:26:41,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601475) roll requested 2024-12-10T07:26:41,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601482 2024-12-10T07:26:41,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601482 2024-12-10T07:26:41,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601482) roll requested 2024-12-10T07:26:41,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601488 2024-12-10T07:26:41,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601488 2024-12-10T07:26:41,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601488) roll requested 2024-12-10T07:26:41,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601496 2024-12-10T07:26:41,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601496 2024-12-10T07:26:41,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601496) roll requested 2024-12-10T07:26:41,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601503 2024-12-10T07:26:41,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601503 2024-12-10T07:26:41,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601503) roll requested 2024-12-10T07:26:41,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601513 2024-12-10T07:26:41,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601513 2024-12-10T07:26:41,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601513) roll requested 2024-12-10T07:26:41,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601521 2024-12-10T07:26:41,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601521 2024-12-10T07:26:41,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601521) roll requested 2024-12-10T07:26:41,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601530 2024-12-10T07:26:41,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601530 2024-12-10T07:26:41,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601530) roll requested 2024-12-10T07:26:41,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601536 2024-12-10T07:26:41,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601536 2024-12-10T07:26:41,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601536) roll requested 2024-12-10T07:26:41,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601543 2024-12-10T07:26:41,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601543 2024-12-10T07:26:41,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601543) roll requested 2024-12-10T07:26:41,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601550 2024-12-10T07:26:41,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601550 2024-12-10T07:26:41,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601550) roll requested 2024-12-10T07:26:41,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601557 2024-12-10T07:26:41,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601557 2024-12-10T07:26:41,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601557) roll requested 2024-12-10T07:26:41,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601566 2024-12-10T07:26:41,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601566 2024-12-10T07:26:41,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601566) roll requested 2024-12-10T07:26:41,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601574 2024-12-10T07:26:41,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601574 2024-12-10T07:26:41,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601574) roll requested 2024-12-10T07:26:41,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601583 2024-12-10T07:26:41,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601583 2024-12-10T07:26:41,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601583) roll requested 2024-12-10T07:26:41,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601593 2024-12-10T07:26:41,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601593 2024-12-10T07:26:41,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601593) roll requested 2024-12-10T07:26:41,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601601 2024-12-10T07:26:41,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601601 2024-12-10T07:26:41,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601601) roll requested 2024-12-10T07:26:41,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601612 2024-12-10T07:26:41,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601612 2024-12-10T07:26:41,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601612) roll requested 2024-12-10T07:26:41,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601621 2024-12-10T07:26:41,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601621 2024-12-10T07:26:41,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601621) roll requested 2024-12-10T07:26:41,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601628 2024-12-10T07:26:41,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601628 2024-12-10T07:26:41,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601628) roll requested 2024-12-10T07:26:41,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601635 2024-12-10T07:26:41,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601635 2024-12-10T07:26:41,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601635) roll requested 2024-12-10T07:26:41,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601644 2024-12-10T07:26:41,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601644 2024-12-10T07:26:41,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601644) roll requested 2024-12-10T07:26:41,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601653 2024-12-10T07:26:41,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601653 2024-12-10T07:26:41,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601653) roll requested 2024-12-10T07:26:41,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601661 2024-12-10T07:26:41,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601661 2024-12-10T07:26:41,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601661) roll requested 2024-12-10T07:26:41,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601672 2024-12-10T07:26:41,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601672 2024-12-10T07:26:41,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601672) roll requested 2024-12-10T07:26:41,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601681 2024-12-10T07:26:41,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601681 2024-12-10T07:26:41,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601681) roll requested 2024-12-10T07:26:41,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601691 2024-12-10T07:26:41,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601691 2024-12-10T07:26:41,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601691) roll requested 2024-12-10T07:26:41,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601699 2024-12-10T07:26:41,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601699 2024-12-10T07:26:41,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601699) roll requested 2024-12-10T07:26:41,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601706 2024-12-10T07:26:41,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601706 2024-12-10T07:26:41,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601706) roll requested 2024-12-10T07:26:41,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601714 2024-12-10T07:26:41,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601714 2024-12-10T07:26:41,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601714) roll requested 2024-12-10T07:26:41,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601722 2024-12-10T07:26:41,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601722 2024-12-10T07:26:41,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601722) roll requested 2024-12-10T07:26:41,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601730 2024-12-10T07:26:41,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601730 2024-12-10T07:26:41,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601730) roll requested 2024-12-10T07:26:41,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601739 2024-12-10T07:26:41,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601739 2024-12-10T07:26:41,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601739) roll requested 2024-12-10T07:26:41,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601750 2024-12-10T07:26:41,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601750 2024-12-10T07:26:41,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601750) roll requested 2024-12-10T07:26:41,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601758 2024-12-10T07:26:41,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601758 2024-12-10T07:26:41,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601758) roll requested 2024-12-10T07:26:41,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601764 2024-12-10T07:26:41,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601764 2024-12-10T07:26:41,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601764) roll requested 2024-12-10T07:26:41,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601772 2024-12-10T07:26:41,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601772 2024-12-10T07:26:41,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601772) roll requested 2024-12-10T07:26:41,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601780 2024-12-10T07:26:41,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601780 2024-12-10T07:26:41,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601780) roll requested 2024-12-10T07:26:41,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601788 2024-12-10T07:26:41,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601788 2024-12-10T07:26:41,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601788) roll requested 2024-12-10T07:26:41,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601796 2024-12-10T07:26:41,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601796 2024-12-10T07:26:41,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601796) roll requested 2024-12-10T07:26:41,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601805 2024-12-10T07:26:41,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601805 2024-12-10T07:26:41,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601805) roll requested 2024-12-10T07:26:41,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601815 2024-12-10T07:26:41,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601815 2024-12-10T07:26:41,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601815) roll requested 2024-12-10T07:26:41,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601823 2024-12-10T07:26:41,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601823 2024-12-10T07:26:41,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601823) roll requested 2024-12-10T07:26:41,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601830 2024-12-10T07:26:41,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601830 2024-12-10T07:26:41,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601830) roll requested 2024-12-10T07:26:41,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601838 2024-12-10T07:26:41,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601838 2024-12-10T07:26:41,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601838) roll requested 2024-12-10T07:26:41,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601845 2024-12-10T07:26:41,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601845 2024-12-10T07:26:41,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601845) roll requested 2024-12-10T07:26:41,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601852 2024-12-10T07:26:41,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601852 2024-12-10T07:26:41,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601852) roll requested 2024-12-10T07:26:41,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601862 2024-12-10T07:26:41,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601862 2024-12-10T07:26:41,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601862) roll requested 2024-12-10T07:26:41,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601872 2024-12-10T07:26:41,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601872 2024-12-10T07:26:41,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601872) roll requested 2024-12-10T07:26:41,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601880 2024-12-10T07:26:41,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601880 2024-12-10T07:26:41,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601880) roll requested 2024-12-10T07:26:41,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601890 2024-12-10T07:26:41,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601890 2024-12-10T07:26:41,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601890) roll requested 2024-12-10T07:26:41,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601900 2024-12-10T07:26:41,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601900 2024-12-10T07:26:41,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601900) roll requested 2024-12-10T07:26:41,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601909 2024-12-10T07:26:41,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601909 2024-12-10T07:26:41,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601909) roll requested 2024-12-10T07:26:41,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601918 2024-12-10T07:26:41,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601918 2024-12-10T07:26:41,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601918) roll requested 2024-12-10T07:26:41,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601928 2024-12-10T07:26:41,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601928 2024-12-10T07:26:41,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601928) roll requested 2024-12-10T07:26:41,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601937 2024-12-10T07:26:41,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601937 2024-12-10T07:26:41,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601937) roll requested 2024-12-10T07:26:41,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601943 2024-12-10T07:26:41,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601943 2024-12-10T07:26:41,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601943) roll requested 2024-12-10T07:26:41,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601950 2024-12-10T07:26:41,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601950 2024-12-10T07:26:41,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601950) roll requested 2024-12-10T07:26:41,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601958 2024-12-10T07:26:41,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601958 2024-12-10T07:26:41,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601958) roll requested 2024-12-10T07:26:41,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601968 2024-12-10T07:26:41,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601968 2024-12-10T07:26:41,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601968) roll requested 2024-12-10T07:26:41,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601977 2024-12-10T07:26:41,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601977 2024-12-10T07:26:41,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601977) roll requested 2024-12-10T07:26:41,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601984 2024-12-10T07:26:41,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:41,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601984 2024-12-10T07:26:41,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:41,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:41,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:41,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:41,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601984) roll requested 2024-12-10T07:26:41,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815601993 2024-12-10T07:26:42,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601993 2024-12-10T07:26:42,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815601993) roll requested 2024-12-10T07:26:42,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602002 2024-12-10T07:26:42,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815601993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602002 2024-12-10T07:26:42,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602002) roll requested 2024-12-10T07:26:42,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602011 2024-12-10T07:26:42,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602011 2024-12-10T07:26:42,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602011) roll requested 2024-12-10T07:26:42,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602021 2024-12-10T07:26:42,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602021 2024-12-10T07:26:42,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602021) roll requested 2024-12-10T07:26:42,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602032 2024-12-10T07:26:42,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602032 2024-12-10T07:26:42,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602032) roll requested 2024-12-10T07:26:42,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602041 2024-12-10T07:26:42,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602041 2024-12-10T07:26:42,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602041) roll requested 2024-12-10T07:26:42,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602052 2024-12-10T07:26:42,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602052 2024-12-10T07:26:42,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602052) roll requested 2024-12-10T07:26:42,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602060 2024-12-10T07:26:42,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602060 2024-12-10T07:26:42,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602060) roll requested 2024-12-10T07:26:42,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602069 2024-12-10T07:26:42,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602069 2024-12-10T07:26:42,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602069) roll requested 2024-12-10T07:26:42,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602078 2024-12-10T07:26:42,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602078 2024-12-10T07:26:42,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602078) roll requested 2024-12-10T07:26:42,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602088 2024-12-10T07:26:42,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602088 2024-12-10T07:26:42,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602088) roll requested 2024-12-10T07:26:42,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602098 2024-12-10T07:26:42,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602098 2024-12-10T07:26:42,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602098) roll requested 2024-12-10T07:26:42,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602107 2024-12-10T07:26:42,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602107 2024-12-10T07:26:42,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602107) roll requested 2024-12-10T07:26:42,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602114 2024-12-10T07:26:42,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602114 2024-12-10T07:26:42,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602114) roll requested 2024-12-10T07:26:42,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602121 2024-12-10T07:26:42,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602121 2024-12-10T07:26:42,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602121) roll requested 2024-12-10T07:26:42,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602130 2024-12-10T07:26:42,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602130 2024-12-10T07:26:42,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602130) roll requested 2024-12-10T07:26:42,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602137 2024-12-10T07:26:42,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602137 2024-12-10T07:26:42,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602137) roll requested 2024-12-10T07:26:42,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602144 2024-12-10T07:26:42,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602144 2024-12-10T07:26:42,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602144) roll requested 2024-12-10T07:26:42,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602153 2024-12-10T07:26:42,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602153 2024-12-10T07:26:42,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602153) roll requested 2024-12-10T07:26:42,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602160 2024-12-10T07:26:42,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602160 2024-12-10T07:26:42,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602160) roll requested 2024-12-10T07:26:42,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602168 2024-12-10T07:26:42,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602168 2024-12-10T07:26:42,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602168) roll requested 2024-12-10T07:26:42,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602175 2024-12-10T07:26:42,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602175 2024-12-10T07:26:42,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602175) roll requested 2024-12-10T07:26:42,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602184 2024-12-10T07:26:42,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602184 2024-12-10T07:26:42,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602184) roll requested 2024-12-10T07:26:42,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602193 2024-12-10T07:26:42,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602193 2024-12-10T07:26:42,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602193) roll requested 2024-12-10T07:26:42,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602202 2024-12-10T07:26:42,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602202 2024-12-10T07:26:42,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602202) roll requested 2024-12-10T07:26:42,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602212 2024-12-10T07:26:42,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602212 2024-12-10T07:26:42,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602212) roll requested 2024-12-10T07:26:42,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602221 2024-12-10T07:26:42,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602221 2024-12-10T07:26:42,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602221) roll requested 2024-12-10T07:26:42,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602231 2024-12-10T07:26:42,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602231 2024-12-10T07:26:42,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602231) roll requested 2024-12-10T07:26:42,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602241 2024-12-10T07:26:42,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602241 2024-12-10T07:26:42,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602241) roll requested 2024-12-10T07:26:42,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602252 2024-12-10T07:26:42,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602252 2024-12-10T07:26:42,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602252) roll requested 2024-12-10T07:26:42,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602262 2024-12-10T07:26:42,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602262 2024-12-10T07:26:42,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602262) roll requested 2024-12-10T07:26:42,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602271 2024-12-10T07:26:42,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602271 2024-12-10T07:26:42,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602271) roll requested 2024-12-10T07:26:42,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602282 2024-12-10T07:26:42,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602282 2024-12-10T07:26:42,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602282) roll requested 2024-12-10T07:26:42,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602292 2024-12-10T07:26:42,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602292 2024-12-10T07:26:42,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602292) roll requested 2024-12-10T07:26:42,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602301 2024-12-10T07:26:42,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602301 2024-12-10T07:26:42,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602301) roll requested 2024-12-10T07:26:42,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602310 2024-12-10T07:26:42,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602310 2024-12-10T07:26:42,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602310) roll requested 2024-12-10T07:26:42,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602319 2024-12-10T07:26:42,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602319 2024-12-10T07:26:42,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602319) roll requested 2024-12-10T07:26:42,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602328 2024-12-10T07:26:42,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602328 2024-12-10T07:26:42,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602328) roll requested 2024-12-10T07:26:42,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602336 2024-12-10T07:26:42,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602336 2024-12-10T07:26:42,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602336) roll requested 2024-12-10T07:26:42,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602346 2024-12-10T07:26:42,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602346 2024-12-10T07:26:42,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602346) roll requested 2024-12-10T07:26:42,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602354 2024-12-10T07:26:42,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602354 2024-12-10T07:26:42,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602354) roll requested 2024-12-10T07:26:42,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602363 2024-12-10T07:26:42,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602363 2024-12-10T07:26:42,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602363) roll requested 2024-12-10T07:26:42,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602373 2024-12-10T07:26:42,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602373 2024-12-10T07:26:42,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602373) roll requested 2024-12-10T07:26:42,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602382 2024-12-10T07:26:42,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602382 2024-12-10T07:26:42,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602382) roll requested 2024-12-10T07:26:42,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602389 2024-12-10T07:26:42,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602389 2024-12-10T07:26:42,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602389) roll requested 2024-12-10T07:26:42,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602396 2024-12-10T07:26:42,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602396 2024-12-10T07:26:42,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602396) roll requested 2024-12-10T07:26:42,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602403 2024-12-10T07:26:42,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602403 2024-12-10T07:26:42,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602403) roll requested 2024-12-10T07:26:42,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602410 2024-12-10T07:26:42,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602410 2024-12-10T07:26:42,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602410) roll requested 2024-12-10T07:26:42,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602419 2024-12-10T07:26:42,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602419 2024-12-10T07:26:42,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602419) roll requested 2024-12-10T07:26:42,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602426 2024-12-10T07:26:42,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602426 2024-12-10T07:26:42,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602426) roll requested 2024-12-10T07:26:42,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602433 2024-12-10T07:26:42,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602433 2024-12-10T07:26:42,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602433) roll requested 2024-12-10T07:26:42,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602441 2024-12-10T07:26:42,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602441 2024-12-10T07:26:42,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602441) roll requested 2024-12-10T07:26:42,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602450 2024-12-10T07:26:42,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602450 2024-12-10T07:26:42,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602450) roll requested 2024-12-10T07:26:42,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602458 2024-12-10T07:26:42,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602458 2024-12-10T07:26:42,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602458) roll requested 2024-12-10T07:26:42,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602466 2024-12-10T07:26:42,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602466 2024-12-10T07:26:42,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602466) roll requested 2024-12-10T07:26:42,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602475 2024-12-10T07:26:42,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602475 2024-12-10T07:26:42,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602475) roll requested 2024-12-10T07:26:42,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602486 2024-12-10T07:26:42,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602486 2024-12-10T07:26:42,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602486) roll requested 2024-12-10T07:26:42,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602495 2024-12-10T07:26:42,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602495 2024-12-10T07:26:42,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602495) roll requested 2024-12-10T07:26:42,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602503 2024-12-10T07:26:42,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602503 2024-12-10T07:26:42,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602503) roll requested 2024-12-10T07:26:42,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602511 2024-12-10T07:26:42,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602511 2024-12-10T07:26:42,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602511) roll requested 2024-12-10T07:26:42,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602521 2024-12-10T07:26:42,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602521 2024-12-10T07:26:42,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602521) roll requested 2024-12-10T07:26:42,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602529 2024-12-10T07:26:42,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602529 2024-12-10T07:26:42,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602529) roll requested 2024-12-10T07:26:42,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602535 2024-12-10T07:26:42,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602535 2024-12-10T07:26:42,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602535) roll requested 2024-12-10T07:26:42,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602542 2024-12-10T07:26:42,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602542 2024-12-10T07:26:42,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602542) roll requested 2024-12-10T07:26:42,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602550 2024-12-10T07:26:42,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602550 2024-12-10T07:26:42,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602550) roll requested 2024-12-10T07:26:42,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602558 2024-12-10T07:26:42,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602558 2024-12-10T07:26:42,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602558) roll requested 2024-12-10T07:26:42,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602567 2024-12-10T07:26:42,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602567 2024-12-10T07:26:42,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602567) roll requested 2024-12-10T07:26:42,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602577 2024-12-10T07:26:42,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602577 2024-12-10T07:26:42,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602577) roll requested 2024-12-10T07:26:42,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602586 2024-12-10T07:26:42,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602586 2024-12-10T07:26:42,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602586) roll requested 2024-12-10T07:26:42,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602594 2024-12-10T07:26:42,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602594 2024-12-10T07:26:42,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602594) roll requested 2024-12-10T07:26:42,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602602 2024-12-10T07:26:42,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602602 2024-12-10T07:26:42,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602602) roll requested 2024-12-10T07:26:42,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602611 2024-12-10T07:26:42,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602611 2024-12-10T07:26:42,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602611) roll requested 2024-12-10T07:26:42,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602620 2024-12-10T07:26:42,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602620 2024-12-10T07:26:42,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602620) roll requested 2024-12-10T07:26:42,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602630 2024-12-10T07:26:42,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602630 2024-12-10T07:26:42,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602630) roll requested 2024-12-10T07:26:42,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602640 2024-12-10T07:26:42,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602640 2024-12-10T07:26:42,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602640) roll requested 2024-12-10T07:26:42,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602648 2024-12-10T07:26:42,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602648 2024-12-10T07:26:42,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602648) roll requested 2024-12-10T07:26:42,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602657 2024-12-10T07:26:42,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602657 2024-12-10T07:26:42,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602657) roll requested 2024-12-10T07:26:42,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602666 2024-12-10T07:26:42,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602666 2024-12-10T07:26:42,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602666) roll requested 2024-12-10T07:26:42,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602673 2024-12-10T07:26:42,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602673 2024-12-10T07:26:42,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602673) roll requested 2024-12-10T07:26:42,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602681 2024-12-10T07:26:42,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602681 2024-12-10T07:26:42,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602681) roll requested 2024-12-10T07:26:42,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602688 2024-12-10T07:26:42,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602688 2024-12-10T07:26:42,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602688) roll requested 2024-12-10T07:26:42,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602693 2024-12-10T07:26:42,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602693 2024-12-10T07:26:42,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602693) roll requested 2024-12-10T07:26:42,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602700 2024-12-10T07:26:42,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602700 2024-12-10T07:26:42,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602700) roll requested 2024-12-10T07:26:42,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602708 2024-12-10T07:26:42,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602708 2024-12-10T07:26:42,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602708) roll requested 2024-12-10T07:26:42,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602716 2024-12-10T07:26:42,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602716 2024-12-10T07:26:42,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602716) roll requested 2024-12-10T07:26:42,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602725 2024-12-10T07:26:42,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602725 2024-12-10T07:26:42,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602725) roll requested 2024-12-10T07:26:42,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602734 2024-12-10T07:26:42,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602734 2024-12-10T07:26:42,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602734) roll requested 2024-12-10T07:26:42,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602745 2024-12-10T07:26:42,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602745 2024-12-10T07:26:42,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602745) roll requested 2024-12-10T07:26:42,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602752 2024-12-10T07:26:42,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602752 2024-12-10T07:26:42,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602752) roll requested 2024-12-10T07:26:42,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602761 2024-12-10T07:26:42,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602761 2024-12-10T07:26:42,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602761) roll requested 2024-12-10T07:26:42,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602769 2024-12-10T07:26:42,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602769 2024-12-10T07:26:42,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602769) roll requested 2024-12-10T07:26:42,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602777 2024-12-10T07:26:42,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602777 2024-12-10T07:26:42,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602777) roll requested 2024-12-10T07:26:42,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602784 2024-12-10T07:26:42,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602784 2024-12-10T07:26:42,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602784) roll requested 2024-12-10T07:26:42,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602792 2024-12-10T07:26:42,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602792 2024-12-10T07:26:42,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602792) roll requested 2024-12-10T07:26:42,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602801 2024-12-10T07:26:42,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602801 2024-12-10T07:26:42,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602801) roll requested 2024-12-10T07:26:42,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602811 2024-12-10T07:26:42,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602811 2024-12-10T07:26:42,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602811) roll requested 2024-12-10T07:26:42,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602819 2024-12-10T07:26:42,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602819 2024-12-10T07:26:42,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602819) roll requested 2024-12-10T07:26:42,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602824 2024-12-10T07:26:42,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602824 2024-12-10T07:26:42,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602824) roll requested 2024-12-10T07:26:42,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602831 2024-12-10T07:26:42,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602831 2024-12-10T07:26:42,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602831) roll requested 2024-12-10T07:26:42,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602840 2024-12-10T07:26:42,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602840 2024-12-10T07:26:42,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602840) roll requested 2024-12-10T07:26:42,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602848 2024-12-10T07:26:42,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602848 2024-12-10T07:26:42,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602848) roll requested 2024-12-10T07:26:42,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602855 2024-12-10T07:26:42,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602855 2024-12-10T07:26:42,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602855) roll requested 2024-12-10T07:26:42,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602865 2024-12-10T07:26:42,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602865 2024-12-10T07:26:42,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602865) roll requested 2024-12-10T07:26:42,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602875 2024-12-10T07:26:42,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602875 2024-12-10T07:26:42,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602875) roll requested 2024-12-10T07:26:42,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602884 2024-12-10T07:26:42,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602884 2024-12-10T07:26:42,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602884) roll requested 2024-12-10T07:26:42,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602891 2024-12-10T07:26:42,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602891 2024-12-10T07:26:42,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602891) roll requested 2024-12-10T07:26:42,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602900 2024-12-10T07:26:42,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602900 2024-12-10T07:26:42,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602900) roll requested 2024-12-10T07:26:42,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602909 2024-12-10T07:26:42,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602909 2024-12-10T07:26:42,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602909) roll requested 2024-12-10T07:26:42,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602919 2024-12-10T07:26:42,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602919 2024-12-10T07:26:42,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602919) roll requested 2024-12-10T07:26:42,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602929 2024-12-10T07:26:42,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602929 2024-12-10T07:26:42,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602929) roll requested 2024-12-10T07:26:42,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602936 2024-12-10T07:26:42,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602936 2024-12-10T07:26:42,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602936) roll requested 2024-12-10T07:26:42,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602944 2024-12-10T07:26:42,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602944 2024-12-10T07:26:42,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602944) roll requested 2024-12-10T07:26:42,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602953 2024-12-10T07:26:42,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602953 2024-12-10T07:26:42,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602953) roll requested 2024-12-10T07:26:42,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602962 2024-12-10T07:26:42,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602962 2024-12-10T07:26:42,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602962) roll requested 2024-12-10T07:26:42,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602969 2024-12-10T07:26:42,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602969 2024-12-10T07:26:42,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602969) roll requested 2024-12-10T07:26:42,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602978 2024-12-10T07:26:42,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602978 2024-12-10T07:26:42,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602978) roll requested 2024-12-10T07:26:42,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602988 2024-12-10T07:26:42,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:42,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602988 2024-12-10T07:26:42,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:42,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:42,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:42,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:42,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602988) roll requested 2024-12-10T07:26:42,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815602996 2024-12-10T07:26:43,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602996 2024-12-10T07:26:43,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815602996) roll requested 2024-12-10T07:26:43,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603006 2024-12-10T07:26:43,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815602996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603006 2024-12-10T07:26:43,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603006) roll requested 2024-12-10T07:26:43,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603014 2024-12-10T07:26:43,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603014 2024-12-10T07:26:43,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603014) roll requested 2024-12-10T07:26:43,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603024 2024-12-10T07:26:43,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603024 2024-12-10T07:26:43,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603024) roll requested 2024-12-10T07:26:43,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603034 2024-12-10T07:26:43,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603034 2024-12-10T07:26:43,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603034) roll requested 2024-12-10T07:26:43,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603043 2024-12-10T07:26:43,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603043 2024-12-10T07:26:43,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603043) roll requested 2024-12-10T07:26:43,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603048 2024-12-10T07:26:43,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603048 2024-12-10T07:26:43,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603048) roll requested 2024-12-10T07:26:43,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603053 2024-12-10T07:26:43,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603053 2024-12-10T07:26:43,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603053) roll requested 2024-12-10T07:26:43,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603061 2024-12-10T07:26:43,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603061 2024-12-10T07:26:43,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603061) roll requested 2024-12-10T07:26:43,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603068 2024-12-10T07:26:43,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603068 2024-12-10T07:26:43,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603068) roll requested 2024-12-10T07:26:43,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603077 2024-12-10T07:26:43,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603077 2024-12-10T07:26:43,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603077) roll requested 2024-12-10T07:26:43,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603087 2024-12-10T07:26:43,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603087 2024-12-10T07:26:43,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603087) roll requested 2024-12-10T07:26:43,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603097 2024-12-10T07:26:43,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603097 2024-12-10T07:26:43,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603097) roll requested 2024-12-10T07:26:43,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603104 2024-12-10T07:26:43,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603104 2024-12-10T07:26:43,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603104) roll requested 2024-12-10T07:26:43,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603112 2024-12-10T07:26:43,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603112 2024-12-10T07:26:43,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603112) roll requested 2024-12-10T07:26:43,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603122 2024-12-10T07:26:43,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603122 2024-12-10T07:26:43,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603122) roll requested 2024-12-10T07:26:43,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603132 2024-12-10T07:26:43,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603132 2024-12-10T07:26:43,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603132) roll requested 2024-12-10T07:26:43,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603140 2024-12-10T07:26:43,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603140 2024-12-10T07:26:43,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603140) roll requested 2024-12-10T07:26:43,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603147 2024-12-10T07:26:43,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603147 2024-12-10T07:26:43,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603147) roll requested 2024-12-10T07:26:43,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603155 2024-12-10T07:26:43,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603155 2024-12-10T07:26:43,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603155) roll requested 2024-12-10T07:26:43,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603163 2024-12-10T07:26:43,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603163 2024-12-10T07:26:43,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603163) roll requested 2024-12-10T07:26:43,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603174 2024-12-10T07:26:43,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603174 2024-12-10T07:26:43,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603174) roll requested 2024-12-10T07:26:43,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603185 2024-12-10T07:26:43,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603185 2024-12-10T07:26:43,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603185) roll requested 2024-12-10T07:26:43,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603193 2024-12-10T07:26:43,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603193 2024-12-10T07:26:43,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603193) roll requested 2024-12-10T07:26:43,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603201 2024-12-10T07:26:43,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603201 2024-12-10T07:26:43,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603201) roll requested 2024-12-10T07:26:43,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603210 2024-12-10T07:26:43,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603210 2024-12-10T07:26:43,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603210) roll requested 2024-12-10T07:26:43,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603219 2024-12-10T07:26:43,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603219 2024-12-10T07:26:43,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603219) roll requested 2024-12-10T07:26:43,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603227 2024-12-10T07:26:43,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603227 2024-12-10T07:26:43,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603227) roll requested 2024-12-10T07:26:43,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603238 2024-12-10T07:26:43,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603238 2024-12-10T07:26:43,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603238) roll requested 2024-12-10T07:26:43,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603245 2024-12-10T07:26:43,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603245 2024-12-10T07:26:43,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603245) roll requested 2024-12-10T07:26:43,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603253 2024-12-10T07:26:43,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603253 2024-12-10T07:26:43,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603253) roll requested 2024-12-10T07:26:43,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603264 2024-12-10T07:26:43,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603264 2024-12-10T07:26:43,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603264) roll requested 2024-12-10T07:26:43,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603270 2024-12-10T07:26:43,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603270 2024-12-10T07:26:43,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603270) roll requested 2024-12-10T07:26:43,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603276 2024-12-10T07:26:43,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603276 2024-12-10T07:26:43,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603276) roll requested 2024-12-10T07:26:43,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603281 2024-12-10T07:26:43,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603281 2024-12-10T07:26:43,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603281) roll requested 2024-12-10T07:26:43,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603287 2024-12-10T07:26:43,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603287 2024-12-10T07:26:43,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603287) roll requested 2024-12-10T07:26:43,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603292 2024-12-10T07:26:43,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603292 2024-12-10T07:26:43,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603292) roll requested 2024-12-10T07:26:43,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603301 2024-12-10T07:26:43,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603301 2024-12-10T07:26:43,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603301) roll requested 2024-12-10T07:26:43,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603307 2024-12-10T07:26:43,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603307 2024-12-10T07:26:43,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603307) roll requested 2024-12-10T07:26:43,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603311 2024-12-10T07:26:43,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603311 2024-12-10T07:26:43,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603311) roll requested 2024-12-10T07:26:43,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603317 2024-12-10T07:26:43,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603317 2024-12-10T07:26:43,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603317) roll requested 2024-12-10T07:26:43,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603323 2024-12-10T07:26:43,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603323 2024-12-10T07:26:43,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603323) roll requested 2024-12-10T07:26:43,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603330 2024-12-10T07:26:43,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603330 2024-12-10T07:26:43,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603330) roll requested 2024-12-10T07:26:43,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603338 2024-12-10T07:26:43,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603338 2024-12-10T07:26:43,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603338) roll requested 2024-12-10T07:26:43,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603347 2024-12-10T07:26:43,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603347 2024-12-10T07:26:43,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603347) roll requested 2024-12-10T07:26:43,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603355 2024-12-10T07:26:43,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603355 2024-12-10T07:26:43,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603355) roll requested 2024-12-10T07:26:43,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603364 2024-12-10T07:26:43,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603364 2024-12-10T07:26:43,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603364) roll requested 2024-12-10T07:26:43,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603373 2024-12-10T07:26:43,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603373 2024-12-10T07:26:43,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603373) roll requested 2024-12-10T07:26:43,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603381 2024-12-10T07:26:43,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603381 2024-12-10T07:26:43,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603381) roll requested 2024-12-10T07:26:43,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603389 2024-12-10T07:26:43,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603389 2024-12-10T07:26:43,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603389) roll requested 2024-12-10T07:26:43,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603398 2024-12-10T07:26:43,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603398 2024-12-10T07:26:43,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603398) roll requested 2024-12-10T07:26:43,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603406 2024-12-10T07:26:43,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603406 2024-12-10T07:26:43,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603406) roll requested 2024-12-10T07:26:43,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603415 2024-12-10T07:26:43,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603415 2024-12-10T07:26:43,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603415) roll requested 2024-12-10T07:26:43,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603424 2024-12-10T07:26:43,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603424 2024-12-10T07:26:43,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603424) roll requested 2024-12-10T07:26:43,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603431 2024-12-10T07:26:43,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603431 2024-12-10T07:26:43,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603431) roll requested 2024-12-10T07:26:43,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603440 2024-12-10T07:26:43,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603440 2024-12-10T07:26:43,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603440) roll requested 2024-12-10T07:26:43,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603450 2024-12-10T07:26:43,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603450 2024-12-10T07:26:43,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603450) roll requested 2024-12-10T07:26:43,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603458 2024-12-10T07:26:43,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603458 2024-12-10T07:26:43,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603458) roll requested 2024-12-10T07:26:43,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603465 2024-12-10T07:26:43,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603465 2024-12-10T07:26:43,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603465) roll requested 2024-12-10T07:26:43,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603473 2024-12-10T07:26:43,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603473 2024-12-10T07:26:43,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603473) roll requested 2024-12-10T07:26:43,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603478 2024-12-10T07:26:43,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603478 2024-12-10T07:26:43,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603478) roll requested 2024-12-10T07:26:43,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603484 2024-12-10T07:26:43,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603484 2024-12-10T07:26:43,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603484) roll requested 2024-12-10T07:26:43,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603492 2024-12-10T07:26:43,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603492 2024-12-10T07:26:43,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603492) roll requested 2024-12-10T07:26:43,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603499 2024-12-10T07:26:43,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603499 2024-12-10T07:26:43,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603499) roll requested 2024-12-10T07:26:43,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603509 2024-12-10T07:26:43,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603509 2024-12-10T07:26:43,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603509) roll requested 2024-12-10T07:26:43,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603518 2024-12-10T07:26:43,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603518 2024-12-10T07:26:43,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603518) roll requested 2024-12-10T07:26:43,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603526 2024-12-10T07:26:43,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603526 2024-12-10T07:26:43,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603526) roll requested 2024-12-10T07:26:43,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603534 2024-12-10T07:26:43,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603534 2024-12-10T07:26:43,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603534) roll requested 2024-12-10T07:26:43,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603542 2024-12-10T07:26:43,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603542 2024-12-10T07:26:43,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603542) roll requested 2024-12-10T07:26:43,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603552 2024-12-10T07:26:43,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603552 2024-12-10T07:26:43,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603552) roll requested 2024-12-10T07:26:43,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603562 2024-12-10T07:26:43,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603562 2024-12-10T07:26:43,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603562) roll requested 2024-12-10T07:26:43,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603571 2024-12-10T07:26:43,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603571 2024-12-10T07:26:43,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603571) roll requested 2024-12-10T07:26:43,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603583 2024-12-10T07:26:43,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603583 2024-12-10T07:26:43,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603583) roll requested 2024-12-10T07:26:43,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603592 2024-12-10T07:26:43,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603592 2024-12-10T07:26:43,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603592) roll requested 2024-12-10T07:26:43,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603600 2024-12-10T07:26:43,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603600 2024-12-10T07:26:43,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603600) roll requested 2024-12-10T07:26:43,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603605 2024-12-10T07:26:43,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603605 2024-12-10T07:26:43,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603605) roll requested 2024-12-10T07:26:43,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603615 2024-12-10T07:26:43,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603615 2024-12-10T07:26:43,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603615) roll requested 2024-12-10T07:26:43,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603623 2024-12-10T07:26:43,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603623 2024-12-10T07:26:43,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603623) roll requested 2024-12-10T07:26:43,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603630 2024-12-10T07:26:43,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603630 2024-12-10T07:26:43,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603630) roll requested 2024-12-10T07:26:43,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603637 2024-12-10T07:26:43,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603637 2024-12-10T07:26:43,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603637) roll requested 2024-12-10T07:26:43,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603646 2024-12-10T07:26:43,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603646 2024-12-10T07:26:43,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603646) roll requested 2024-12-10T07:26:43,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603654 2024-12-10T07:26:43,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603654 2024-12-10T07:26:43,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603654) roll requested 2024-12-10T07:26:43,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603664 2024-12-10T07:26:43,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603664 2024-12-10T07:26:43,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603664) roll requested 2024-12-10T07:26:43,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603671 2024-12-10T07:26:43,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603671 2024-12-10T07:26:43,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603671) roll requested 2024-12-10T07:26:43,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603677 2024-12-10T07:26:43,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603677 2024-12-10T07:26:43,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603677) roll requested 2024-12-10T07:26:43,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603683 2024-12-10T07:26:43,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603683 2024-12-10T07:26:43,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603683) roll requested 2024-12-10T07:26:43,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603691 2024-12-10T07:26:43,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603691 2024-12-10T07:26:43,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603691) roll requested 2024-12-10T07:26:43,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603698 2024-12-10T07:26:43,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603698 2024-12-10T07:26:43,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603698) roll requested 2024-12-10T07:26:43,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603707 2024-12-10T07:26:43,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603707 2024-12-10T07:26:43,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603707) roll requested 2024-12-10T07:26:43,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603716 2024-12-10T07:26:43,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603716 2024-12-10T07:26:43,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603716) roll requested 2024-12-10T07:26:43,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603725 2024-12-10T07:26:43,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603725 2024-12-10T07:26:43,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603725) roll requested 2024-12-10T07:26:43,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603733 2024-12-10T07:26:43,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603733 2024-12-10T07:26:43,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603733) roll requested 2024-12-10T07:26:43,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603741 2024-12-10T07:26:43,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603741 2024-12-10T07:26:43,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603741) roll requested 2024-12-10T07:26:43,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603748 2024-12-10T07:26:43,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603748 2024-12-10T07:26:43,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603748) roll requested 2024-12-10T07:26:43,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603756 2024-12-10T07:26:43,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603756 2024-12-10T07:26:43,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603756) roll requested 2024-12-10T07:26:43,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603765 2024-12-10T07:26:43,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603765 2024-12-10T07:26:43,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603765) roll requested 2024-12-10T07:26:43,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603774 2024-12-10T07:26:43,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603774 2024-12-10T07:26:43,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603774) roll requested 2024-12-10T07:26:43,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603780 2024-12-10T07:26:43,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603780 2024-12-10T07:26:43,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603780) roll requested 2024-12-10T07:26:43,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603787 2024-12-10T07:26:43,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603787 2024-12-10T07:26:43,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603787) roll requested 2024-12-10T07:26:43,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603795 2024-12-10T07:26:43,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603795 2024-12-10T07:26:43,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603795) roll requested 2024-12-10T07:26:43,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603803 2024-12-10T07:26:43,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603803 2024-12-10T07:26:43,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603803) roll requested 2024-12-10T07:26:43,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603811 2024-12-10T07:26:43,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603811 2024-12-10T07:26:43,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603811) roll requested 2024-12-10T07:26:43,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603820 2024-12-10T07:26:43,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603820 2024-12-10T07:26:43,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603820) roll requested 2024-12-10T07:26:43,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603828 2024-12-10T07:26:43,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603828 2024-12-10T07:26:43,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603828) roll requested 2024-12-10T07:26:43,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603837 2024-12-10T07:26:43,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603837 2024-12-10T07:26:43,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603837) roll requested 2024-12-10T07:26:43,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603846 2024-12-10T07:26:43,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603846 2024-12-10T07:26:43,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603846) roll requested 2024-12-10T07:26:43,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603854 2024-12-10T07:26:43,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603854 2024-12-10T07:26:43,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603854) roll requested 2024-12-10T07:26:43,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603864 2024-12-10T07:26:43,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603864 2024-12-10T07:26:43,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603864) roll requested 2024-12-10T07:26:43,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603874 2024-12-10T07:26:43,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603874 2024-12-10T07:26:43,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603874) roll requested 2024-12-10T07:26:43,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603880 2024-12-10T07:26:43,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603880 2024-12-10T07:26:43,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603880) roll requested 2024-12-10T07:26:43,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603886 2024-12-10T07:26:43,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603886 2024-12-10T07:26:43,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603886) roll requested 2024-12-10T07:26:43,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603893 2024-12-10T07:26:43,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603893 2024-12-10T07:26:43,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603893) roll requested 2024-12-10T07:26:43,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603900 2024-12-10T07:26:43,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603900 2024-12-10T07:26:43,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603900) roll requested 2024-12-10T07:26:43,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603907 2024-12-10T07:26:43,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603907 2024-12-10T07:26:43,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603907) roll requested 2024-12-10T07:26:43,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603916 2024-12-10T07:26:43,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603916 2024-12-10T07:26:43,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603916) roll requested 2024-12-10T07:26:43,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603925 2024-12-10T07:26:43,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603925 2024-12-10T07:26:43,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603925) roll requested 2024-12-10T07:26:43,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603936 2024-12-10T07:26:43,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603936 2024-12-10T07:26:43,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603936) roll requested 2024-12-10T07:26:43,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603946 2024-12-10T07:26:43,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603946 2024-12-10T07:26:43,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603946) roll requested 2024-12-10T07:26:43,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603956 2024-12-10T07:26:43,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603956 2024-12-10T07:26:43,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603956) roll requested 2024-12-10T07:26:43,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603964 2024-12-10T07:26:43,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603964 2024-12-10T07:26:43,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603964) roll requested 2024-12-10T07:26:43,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603972 2024-12-10T07:26:43,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603972 2024-12-10T07:26:43,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603972) roll requested 2024-12-10T07:26:43,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603980 2024-12-10T07:26:43,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603980 2024-12-10T07:26:43,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603980) roll requested 2024-12-10T07:26:43,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603989 2024-12-10T07:26:43,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:43,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603989 2024-12-10T07:26:43,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:43,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:43,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:43,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:43,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603989) roll requested 2024-12-10T07:26:43,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815603997 2024-12-10T07:26:44,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603997 2024-12-10T07:26:44,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815603997) roll requested 2024-12-10T07:26:44,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604006 2024-12-10T07:26:44,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815603997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604006 2024-12-10T07:26:44,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604006) roll requested 2024-12-10T07:26:44,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604015 2024-12-10T07:26:44,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604015 2024-12-10T07:26:44,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604015) roll requested 2024-12-10T07:26:44,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604023 2024-12-10T07:26:44,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604023 2024-12-10T07:26:44,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604023) roll requested 2024-12-10T07:26:44,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604033 2024-12-10T07:26:44,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604033 2024-12-10T07:26:44,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604033) roll requested 2024-12-10T07:26:44,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604041 2024-12-10T07:26:44,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604041 2024-12-10T07:26:44,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604041) roll requested 2024-12-10T07:26:44,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604048 2024-12-10T07:26:44,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604048 2024-12-10T07:26:44,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604048) roll requested 2024-12-10T07:26:44,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604056 2024-12-10T07:26:44,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604056 2024-12-10T07:26:44,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604056) roll requested 2024-12-10T07:26:44,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604069 2024-12-10T07:26:44,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604069 2024-12-10T07:26:44,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604069) roll requested 2024-12-10T07:26:44,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604076 2024-12-10T07:26:44,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604076 2024-12-10T07:26:44,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604076) roll requested 2024-12-10T07:26:44,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604081 2024-12-10T07:26:44,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604081 2024-12-10T07:26:44,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604081) roll requested 2024-12-10T07:26:44,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604086 2024-12-10T07:26:44,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604086 2024-12-10T07:26:44,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604086) roll requested 2024-12-10T07:26:44,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604092 2024-12-10T07:26:44,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604092 2024-12-10T07:26:44,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604092) roll requested 2024-12-10T07:26:44,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604099 2024-12-10T07:26:44,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604099 2024-12-10T07:26:44,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604099) roll requested 2024-12-10T07:26:44,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604108 2024-12-10T07:26:44,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604108 2024-12-10T07:26:44,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604108) roll requested 2024-12-10T07:26:44,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604118 2024-12-10T07:26:44,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604118 2024-12-10T07:26:44,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604118) roll requested 2024-12-10T07:26:44,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604127 2024-12-10T07:26:44,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604127 2024-12-10T07:26:44,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604127) roll requested 2024-12-10T07:26:44,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604137 2024-12-10T07:26:44,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604137 2024-12-10T07:26:44,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604137) roll requested 2024-12-10T07:26:44,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604146 2024-12-10T07:26:44,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604146 2024-12-10T07:26:44,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604146) roll requested 2024-12-10T07:26:44,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604153 2024-12-10T07:26:44,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604153 2024-12-10T07:26:44,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604153) roll requested 2024-12-10T07:26:44,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604161 2024-12-10T07:26:44,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604161 2024-12-10T07:26:44,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604161) roll requested 2024-12-10T07:26:44,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604169 2024-12-10T07:26:44,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604169 2024-12-10T07:26:44,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604169) roll requested 2024-12-10T07:26:44,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604177 2024-12-10T07:26:44,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604177 2024-12-10T07:26:44,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604177) roll requested 2024-12-10T07:26:44,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604184 2024-12-10T07:26:44,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604184 2024-12-10T07:26:44,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604184) roll requested 2024-12-10T07:26:44,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604190 2024-12-10T07:26:44,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604190 2024-12-10T07:26:44,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604190) roll requested 2024-12-10T07:26:44,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604195 2024-12-10T07:26:44,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604195 2024-12-10T07:26:44,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604195) roll requested 2024-12-10T07:26:44,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604200 2024-12-10T07:26:44,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604200 2024-12-10T07:26:44,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604200) roll requested 2024-12-10T07:26:44,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604207 2024-12-10T07:26:44,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604207 2024-12-10T07:26:44,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604207) roll requested 2024-12-10T07:26:44,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604214 2024-12-10T07:26:44,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604214 2024-12-10T07:26:44,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604214) roll requested 2024-12-10T07:26:44,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604222 2024-12-10T07:26:44,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604222 2024-12-10T07:26:44,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604222) roll requested 2024-12-10T07:26:44,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604229 2024-12-10T07:26:44,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604229 2024-12-10T07:26:44,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604229) roll requested 2024-12-10T07:26:44,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604238 2024-12-10T07:26:44,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604238 2024-12-10T07:26:44,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604238) roll requested 2024-12-10T07:26:44,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604246 2024-12-10T07:26:44,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604246 2024-12-10T07:26:44,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604246) roll requested 2024-12-10T07:26:44,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604253 2024-12-10T07:26:44,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604253 2024-12-10T07:26:44,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604253) roll requested 2024-12-10T07:26:44,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604261 2024-12-10T07:26:44,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604261 2024-12-10T07:26:44,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604261) roll requested 2024-12-10T07:26:44,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604270 2024-12-10T07:26:44,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604270 2024-12-10T07:26:44,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604270) roll requested 2024-12-10T07:26:44,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604277 2024-12-10T07:26:44,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604277 2024-12-10T07:26:44,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604277) roll requested 2024-12-10T07:26:44,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604285 2024-12-10T07:26:44,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604285 2024-12-10T07:26:44,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604285) roll requested 2024-12-10T07:26:44,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604294 2024-12-10T07:26:44,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604294 2024-12-10T07:26:44,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604294) roll requested 2024-12-10T07:26:44,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604304 2024-12-10T07:26:44,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604304 2024-12-10T07:26:44,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604304) roll requested 2024-12-10T07:26:44,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604314 2024-12-10T07:26:44,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604314 2024-12-10T07:26:44,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604314) roll requested 2024-12-10T07:26:44,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604324 2024-12-10T07:26:44,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604324 2024-12-10T07:26:44,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604324) roll requested 2024-12-10T07:26:44,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604333 2024-12-10T07:26:44,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604333 2024-12-10T07:26:44,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604333) roll requested 2024-12-10T07:26:44,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604341 2024-12-10T07:26:44,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604341 2024-12-10T07:26:44,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604341) roll requested 2024-12-10T07:26:44,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604351 2024-12-10T07:26:44,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604351 2024-12-10T07:26:44,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604351) roll requested 2024-12-10T07:26:44,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604358 2024-12-10T07:26:44,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604358 2024-12-10T07:26:44,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604358) roll requested 2024-12-10T07:26:44,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604365 2024-12-10T07:26:44,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604365 2024-12-10T07:26:44,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604365) roll requested 2024-12-10T07:26:44,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604373 2024-12-10T07:26:44,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604373 2024-12-10T07:26:44,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604373) roll requested 2024-12-10T07:26:44,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604381 2024-12-10T07:26:44,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604381 2024-12-10T07:26:44,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604381) roll requested 2024-12-10T07:26:44,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604389 2024-12-10T07:26:44,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604389 2024-12-10T07:26:44,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604389) roll requested 2024-12-10T07:26:44,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604396 2024-12-10T07:26:44,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604396 2024-12-10T07:26:44,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604396) roll requested 2024-12-10T07:26:44,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604404 2024-12-10T07:26:44,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604404 2024-12-10T07:26:44,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604404) roll requested 2024-12-10T07:26:44,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604413 2024-12-10T07:26:44,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604413 2024-12-10T07:26:44,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604413) roll requested 2024-12-10T07:26:44,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604423 2024-12-10T07:26:44,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604423 2024-12-10T07:26:44,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604423) roll requested 2024-12-10T07:26:44,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604429 2024-12-10T07:26:44,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604429 2024-12-10T07:26:44,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604429) roll requested 2024-12-10T07:26:44,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604436 2024-12-10T07:26:44,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604436 2024-12-10T07:26:44,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604436) roll requested 2024-12-10T07:26:44,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604446 2024-12-10T07:26:44,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604446 2024-12-10T07:26:44,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604446) roll requested 2024-12-10T07:26:44,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604456 2024-12-10T07:26:44,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604456 2024-12-10T07:26:44,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604456) roll requested 2024-12-10T07:26:44,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604464 2024-12-10T07:26:44,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604464 2024-12-10T07:26:44,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604464) roll requested 2024-12-10T07:26:44,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604473 2024-12-10T07:26:44,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604473 2024-12-10T07:26:44,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604473) roll requested 2024-12-10T07:26:44,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604482 2024-12-10T07:26:44,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604482 2024-12-10T07:26:44,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604482) roll requested 2024-12-10T07:26:44,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604492 2024-12-10T07:26:44,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604492 2024-12-10T07:26:44,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,502 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604492) roll requested 2024-12-10T07:26:44,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604503 2024-12-10T07:26:44,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604503 2024-12-10T07:26:44,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604503) roll requested 2024-12-10T07:26:44,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604509 2024-12-10T07:26:44,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604509 2024-12-10T07:26:44,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604509) roll requested 2024-12-10T07:26:44,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604516 2024-12-10T07:26:44,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604516 2024-12-10T07:26:44,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604516) roll requested 2024-12-10T07:26:44,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604524 2024-12-10T07:26:44,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604524 2024-12-10T07:26:44,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604524) roll requested 2024-12-10T07:26:44,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604533 2024-12-10T07:26:44,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604533 2024-12-10T07:26:44,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604533) roll requested 2024-12-10T07:26:44,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604541 2024-12-10T07:26:44,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604541 2024-12-10T07:26:44,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604541) roll requested 2024-12-10T07:26:44,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604547 2024-12-10T07:26:44,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604547 2024-12-10T07:26:44,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604547) roll requested 2024-12-10T07:26:44,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604555 2024-12-10T07:26:44,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604555 2024-12-10T07:26:44,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604555) roll requested 2024-12-10T07:26:44,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604565 2024-12-10T07:26:44,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604565 2024-12-10T07:26:44,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604565) roll requested 2024-12-10T07:26:44,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604576 2024-12-10T07:26:44,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604576 2024-12-10T07:26:44,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604576) roll requested 2024-12-10T07:26:44,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604582 2024-12-10T07:26:44,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604582 2024-12-10T07:26:44,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604582) roll requested 2024-12-10T07:26:44,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604589 2024-12-10T07:26:44,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604589 2024-12-10T07:26:44,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604589) roll requested 2024-12-10T07:26:44,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604598 2024-12-10T07:26:44,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604598 2024-12-10T07:26:44,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604598) roll requested 2024-12-10T07:26:44,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604605 2024-12-10T07:26:44,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604605 2024-12-10T07:26:44,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604605) roll requested 2024-12-10T07:26:44,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604613 2024-12-10T07:26:44,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604613 2024-12-10T07:26:44,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604613) roll requested 2024-12-10T07:26:44,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604621 2024-12-10T07:26:44,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604621 2024-12-10T07:26:44,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604621) roll requested 2024-12-10T07:26:44,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604629 2024-12-10T07:26:44,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604629 2024-12-10T07:26:44,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604629) roll requested 2024-12-10T07:26:44,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604638 2024-12-10T07:26:44,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604638 2024-12-10T07:26:44,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604638) roll requested 2024-12-10T07:26:44,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604646 2024-12-10T07:26:44,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604646 2024-12-10T07:26:44,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604646) roll requested 2024-12-10T07:26:44,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604654 2024-12-10T07:26:44,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604654 2024-12-10T07:26:44,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604654) roll requested 2024-12-10T07:26:44,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604662 2024-12-10T07:26:44,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604662 2024-12-10T07:26:44,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604662) roll requested 2024-12-10T07:26:44,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604671 2024-12-10T07:26:44,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604671 2024-12-10T07:26:44,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604671) roll requested 2024-12-10T07:26:44,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604678 2024-12-10T07:26:44,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604678 2024-12-10T07:26:44,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604678) roll requested 2024-12-10T07:26:44,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604686 2024-12-10T07:26:44,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604686 2024-12-10T07:26:44,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604686) roll requested 2024-12-10T07:26:44,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604696 2024-12-10T07:26:44,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604696 2024-12-10T07:26:44,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604696) roll requested 2024-12-10T07:26:44,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604705 2024-12-10T07:26:44,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604705 2024-12-10T07:26:44,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604705) roll requested 2024-12-10T07:26:44,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604716 2024-12-10T07:26:44,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604716 2024-12-10T07:26:44,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604716) roll requested 2024-12-10T07:26:44,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604727 2024-12-10T07:26:44,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604727 2024-12-10T07:26:44,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604727) roll requested 2024-12-10T07:26:44,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604733 2024-12-10T07:26:44,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604733 2024-12-10T07:26:44,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604733) roll requested 2024-12-10T07:26:44,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604741 2024-12-10T07:26:44,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604741 2024-12-10T07:26:44,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604741) roll requested 2024-12-10T07:26:44,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604752 2024-12-10T07:26:44,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604752 2024-12-10T07:26:44,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604752) roll requested 2024-12-10T07:26:44,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604761 2024-12-10T07:26:44,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604761 2024-12-10T07:26:44,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604761) roll requested 2024-12-10T07:26:44,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604773 2024-12-10T07:26:44,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604773 2024-12-10T07:26:44,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604773) roll requested 2024-12-10T07:26:44,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604779 2024-12-10T07:26:44,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604779 2024-12-10T07:26:44,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604779) roll requested 2024-12-10T07:26:44,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604785 2024-12-10T07:26:44,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604785 2024-12-10T07:26:44,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604785) roll requested 2024-12-10T07:26:44,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604791 2024-12-10T07:26:44,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604791 2024-12-10T07:26:44,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604791) roll requested 2024-12-10T07:26:44,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604797 2024-12-10T07:26:44,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604797 2024-12-10T07:26:44,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604797) roll requested 2024-12-10T07:26:44,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604802 2024-12-10T07:26:44,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604802 2024-12-10T07:26:44,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604802) roll requested 2024-12-10T07:26:44,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604808 2024-12-10T07:26:44,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604808 2024-12-10T07:26:44,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604808) roll requested 2024-12-10T07:26:44,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604814 2024-12-10T07:26:44,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604814 2024-12-10T07:26:44,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604814) roll requested 2024-12-10T07:26:44,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604820 2024-12-10T07:26:44,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604820 2024-12-10T07:26:44,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604820) roll requested 2024-12-10T07:26:44,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604825 2024-12-10T07:26:44,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604825 2024-12-10T07:26:44,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604825) roll requested 2024-12-10T07:26:44,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604831 2024-12-10T07:26:44,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604831 2024-12-10T07:26:44,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604831) roll requested 2024-12-10T07:26:44,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604837 2024-12-10T07:26:44,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604837 2024-12-10T07:26:44,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604837) roll requested 2024-12-10T07:26:44,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604843 2024-12-10T07:26:44,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604843 2024-12-10T07:26:44,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604843) roll requested 2024-12-10T07:26:44,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604848 2024-12-10T07:26:44,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604848 2024-12-10T07:26:44,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604848) roll requested 2024-12-10T07:26:44,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604853 2024-12-10T07:26:44,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604853 2024-12-10T07:26:44,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604853) roll requested 2024-12-10T07:26:44,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604858 2024-12-10T07:26:44,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604858 2024-12-10T07:26:44,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604858) roll requested 2024-12-10T07:26:44,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604865 2024-12-10T07:26:44,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604865 2024-12-10T07:26:44,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604865) roll requested 2024-12-10T07:26:44,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604872 2024-12-10T07:26:44,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604872 2024-12-10T07:26:44,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604872) roll requested 2024-12-10T07:26:44,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604880 2024-12-10T07:26:44,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604880 2024-12-10T07:26:44,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604880) roll requested 2024-12-10T07:26:44,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604888 2024-12-10T07:26:44,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604888 2024-12-10T07:26:44,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604888) roll requested 2024-12-10T07:26:44,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604896 2024-12-10T07:26:44,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604896 2024-12-10T07:26:44,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604896) roll requested 2024-12-10T07:26:44,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604904 2024-12-10T07:26:44,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604904 2024-12-10T07:26:44,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604904) roll requested 2024-12-10T07:26:44,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604913 2024-12-10T07:26:44,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604913 2024-12-10T07:26:44,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604913) roll requested 2024-12-10T07:26:44,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604924 2024-12-10T07:26:44,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604924 2024-12-10T07:26:44,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604924) roll requested 2024-12-10T07:26:44,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604934 2024-12-10T07:26:44,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604934 2024-12-10T07:26:44,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604934) roll requested 2024-12-10T07:26:44,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604944 2024-12-10T07:26:44,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604944 2024-12-10T07:26:44,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604944) roll requested 2024-12-10T07:26:44,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604951 2024-12-10T07:26:44,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604951 2024-12-10T07:26:44,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604951) roll requested 2024-12-10T07:26:44,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604960 2024-12-10T07:26:44,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604960 2024-12-10T07:26:44,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604960) roll requested 2024-12-10T07:26:44,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604969 2024-12-10T07:26:44,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604969 2024-12-10T07:26:44,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604969) roll requested 2024-12-10T07:26:44,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604978 2024-12-10T07:26:44,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604978 2024-12-10T07:26:44,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604978) roll requested 2024-12-10T07:26:44,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604983 2024-12-10T07:26:44,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604983 2024-12-10T07:26:44,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604983) roll requested 2024-12-10T07:26:44,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604989 2024-12-10T07:26:44,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:44,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604989 2024-12-10T07:26:44,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:44,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:44,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:44,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:44,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604989) roll requested 2024-12-10T07:26:44,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815604997 2024-12-10T07:26:45,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604997 2024-12-10T07:26:45,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815604997) roll requested 2024-12-10T07:26:45,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605004 2024-12-10T07:26:45,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815604997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605004 2024-12-10T07:26:45,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605004) roll requested 2024-12-10T07:26:45,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605011 2024-12-10T07:26:45,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605011 2024-12-10T07:26:45,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605011) roll requested 2024-12-10T07:26:45,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605020 2024-12-10T07:26:45,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605020 2024-12-10T07:26:45,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605020) roll requested 2024-12-10T07:26:45,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605028 2024-12-10T07:26:45,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605028 2024-12-10T07:26:45,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605028) roll requested 2024-12-10T07:26:45,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605038 2024-12-10T07:26:45,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605038 2024-12-10T07:26:45,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605038) roll requested 2024-12-10T07:26:45,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605048 2024-12-10T07:26:45,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605048 2024-12-10T07:26:45,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605048) roll requested 2024-12-10T07:26:45,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605058 2024-12-10T07:26:45,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605058 2024-12-10T07:26:45,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605058) roll requested 2024-12-10T07:26:45,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605064 2024-12-10T07:26:45,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605064 2024-12-10T07:26:45,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605064) roll requested 2024-12-10T07:26:45,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605071 2024-12-10T07:26:45,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605071 2024-12-10T07:26:45,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605071) roll requested 2024-12-10T07:26:45,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605078 2024-12-10T07:26:45,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605078 2024-12-10T07:26:45,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605078) roll requested 2024-12-10T07:26:45,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605087 2024-12-10T07:26:45,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605087 2024-12-10T07:26:45,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605087) roll requested 2024-12-10T07:26:45,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605150 2024-12-10T07:26:45,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605150 2024-12-10T07:26:45,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605150) roll requested 2024-12-10T07:26:45,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605157 2024-12-10T07:26:45,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605157 2024-12-10T07:26:45,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605157) roll requested 2024-12-10T07:26:45,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605165 2024-12-10T07:26:45,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605165 2024-12-10T07:26:45,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605165) roll requested 2024-12-10T07:26:45,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605174 2024-12-10T07:26:45,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605174 2024-12-10T07:26:45,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605174) roll requested 2024-12-10T07:26:45,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605183 2024-12-10T07:26:45,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605183 2024-12-10T07:26:45,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605183) roll requested 2024-12-10T07:26:45,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605192 2024-12-10T07:26:45,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605192 2024-12-10T07:26:45,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605192) roll requested 2024-12-10T07:26:45,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605199 2024-12-10T07:26:45,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605199 2024-12-10T07:26:45,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605199) roll requested 2024-12-10T07:26:45,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605207 2024-12-10T07:26:45,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605207 2024-12-10T07:26:45,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605207) roll requested 2024-12-10T07:26:45,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605215 2024-12-10T07:26:45,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605215 2024-12-10T07:26:45,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605215) roll requested 2024-12-10T07:26:45,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605224 2024-12-10T07:26:45,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605224 2024-12-10T07:26:45,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605224) roll requested 2024-12-10T07:26:45,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605232 2024-12-10T07:26:45,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605232 2024-12-10T07:26:45,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605232) roll requested 2024-12-10T07:26:45,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605242 2024-12-10T07:26:45,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605242 2024-12-10T07:26:45,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605242) roll requested 2024-12-10T07:26:45,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605250 2024-12-10T07:26:45,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605250 2024-12-10T07:26:45,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605250) roll requested 2024-12-10T07:26:45,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605258 2024-12-10T07:26:45,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605258 2024-12-10T07:26:45,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605258) roll requested 2024-12-10T07:26:45,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605266 2024-12-10T07:26:45,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605266 2024-12-10T07:26:45,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605266) roll requested 2024-12-10T07:26:45,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605275 2024-12-10T07:26:45,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605275 2024-12-10T07:26:45,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605275) roll requested 2024-12-10T07:26:45,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605285 2024-12-10T07:26:45,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605285 2024-12-10T07:26:45,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605285) roll requested 2024-12-10T07:26:45,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605296 2024-12-10T07:26:45,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605296 2024-12-10T07:26:45,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605296) roll requested 2024-12-10T07:26:45,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605309 2024-12-10T07:26:45,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605309 2024-12-10T07:26:45,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605309) roll requested 2024-12-10T07:26:45,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605321 2024-12-10T07:26:45,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605321 2024-12-10T07:26:45,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605321) roll requested 2024-12-10T07:26:45,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605327 2024-12-10T07:26:45,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605327 2024-12-10T07:26:45,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605327) roll requested 2024-12-10T07:26:45,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605334 2024-12-10T07:26:45,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605334 2024-12-10T07:26:45,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605334) roll requested 2024-12-10T07:26:45,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605340 2024-12-10T07:26:45,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605340 2024-12-10T07:26:45,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605340) roll requested 2024-12-10T07:26:45,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605345 2024-12-10T07:26:45,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605345 2024-12-10T07:26:45,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605345) roll requested 2024-12-10T07:26:45,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605354 2024-12-10T07:26:45,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605354 2024-12-10T07:26:45,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605354) roll requested 2024-12-10T07:26:45,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605359 2024-12-10T07:26:45,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605359 2024-12-10T07:26:45,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605359) roll requested 2024-12-10T07:26:45,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605366 2024-12-10T07:26:45,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605366 2024-12-10T07:26:45,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605366) roll requested 2024-12-10T07:26:45,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605372 2024-12-10T07:26:45,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605372 2024-12-10T07:26:45,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605372) roll requested 2024-12-10T07:26:45,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605377 2024-12-10T07:26:45,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605372 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605377 2024-12-10T07:26:45,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605377) roll requested 2024-12-10T07:26:45,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605383 2024-12-10T07:26:45,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605383 2024-12-10T07:26:45,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605383) roll requested 2024-12-10T07:26:45,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605393 2024-12-10T07:26:45,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605393 2024-12-10T07:26:45,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605393) roll requested 2024-12-10T07:26:45,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605401 2024-12-10T07:26:45,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605401 2024-12-10T07:26:45,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605401) roll requested 2024-12-10T07:26:45,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605408 2024-12-10T07:26:45,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605408 2024-12-10T07:26:45,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605408) roll requested 2024-12-10T07:26:45,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605414 2024-12-10T07:26:45,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605414 2024-12-10T07:26:45,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605414) roll requested 2024-12-10T07:26:45,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605421 2024-12-10T07:26:45,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605421 2024-12-10T07:26:45,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605421) roll requested 2024-12-10T07:26:45,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605427 2024-12-10T07:26:45,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605427 2024-12-10T07:26:45,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605427) roll requested 2024-12-10T07:26:45,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605433 2024-12-10T07:26:45,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605433 2024-12-10T07:26:45,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605433) roll requested 2024-12-10T07:26:45,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605439 2024-12-10T07:26:45,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605439 2024-12-10T07:26:45,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605439) roll requested 2024-12-10T07:26:45,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605445 2024-12-10T07:26:45,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605445 2024-12-10T07:26:45,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605445) roll requested 2024-12-10T07:26:45,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605451 2024-12-10T07:26:45,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605451 2024-12-10T07:26:45,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605451) roll requested 2024-12-10T07:26:45,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605457 2024-12-10T07:26:45,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605457 2024-12-10T07:26:45,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605457) roll requested 2024-12-10T07:26:45,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605463 2024-12-10T07:26:45,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605463 2024-12-10T07:26:45,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605463) roll requested 2024-12-10T07:26:45,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605469 2024-12-10T07:26:45,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605469 2024-12-10T07:26:45,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605469) roll requested 2024-12-10T07:26:45,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605476 2024-12-10T07:26:45,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605476 2024-12-10T07:26:45,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605476) roll requested 2024-12-10T07:26:45,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605484 2024-12-10T07:26:45,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605484 2024-12-10T07:26:45,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605484) roll requested 2024-12-10T07:26:45,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605492 2024-12-10T07:26:45,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605492 2024-12-10T07:26:45,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605492) roll requested 2024-12-10T07:26:45,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605501 2024-12-10T07:26:45,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605501 2024-12-10T07:26:45,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605501) roll requested 2024-12-10T07:26:45,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605511 2024-12-10T07:26:45,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605511 2024-12-10T07:26:45,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605511) roll requested 2024-12-10T07:26:45,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605520 2024-12-10T07:26:45,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605520 2024-12-10T07:26:45,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605520) roll requested 2024-12-10T07:26:45,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605527 2024-12-10T07:26:45,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605527 2024-12-10T07:26:45,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605527) roll requested 2024-12-10T07:26:45,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605535 2024-12-10T07:26:45,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605535 2024-12-10T07:26:45,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605535) roll requested 2024-12-10T07:26:45,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605543 2024-12-10T07:26:45,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605543 2024-12-10T07:26:45,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605543) roll requested 2024-12-10T07:26:45,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605553 2024-12-10T07:26:45,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605553 2024-12-10T07:26:45,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605553) roll requested 2024-12-10T07:26:45,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605563 2024-12-10T07:26:45,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605563 2024-12-10T07:26:45,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605563) roll requested 2024-12-10T07:26:45,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605573 2024-12-10T07:26:45,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605573 2024-12-10T07:26:45,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605573) roll requested 2024-12-10T07:26:45,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605580 2024-12-10T07:26:45,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605580 2024-12-10T07:26:45,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T07:26:45,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T07:26:45,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): 493d0679b9ca3cdf33d820337051847c[MyCF] 2024-12-10T07:26:45,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 493d0679b9ca3cdf33d820337051847c, because it is not online on us 2024-12-10T07:26:45,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733815605580) roll requested 2024-12-10T07:26:45,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733815605590 2024-12-10T07:26:45,590 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:26:45,594 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 493d0679b9ca3cdf33d820337051847c, disabling compactions & flushes 2024-12-10T07:26:45,594 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1733815575321.493d0679b9ca3cdf33d820337051847c. 2024-12-10T07:26:45,594 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1733815575321.493d0679b9ca3cdf33d820337051847c. 2024-12-10T07:26:45,595 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1733815575321.493d0679b9ca3cdf33d820337051847c. after waiting 0 ms 2024-12-10T07:26:45,595 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1733815575321.493d0679b9ca3cdf33d820337051847c. 2024-12-10T07:26:45,597 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1733815575321.493d0679b9ca3cdf33d820337051847c. 2024-12-10T07:26:45,597 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1733815575321.493d0679b9ca3cdf33d820337051847c. 2024-12-10T07:26:45,597 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 493d0679b9ca3cdf33d820337051847c: Waiting for close lock at 1733815605594Disabling compacts and flushes for region at 1733815605594Disabling writes for close at 1733815605595 (+1 ms)Writing region close event to WAL at 1733815605596 (+1 ms)Closed at 1733815605597 (+1 ms) 2024-12-10T07:26:45,596 WARN [LogRoller {}] wal.AbstractProtobufLogWriter(199): Init output failed, path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733815605590 java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 27 more 2024-12-10T07:26:45,597 DEBUG [LogRoller {}] wal.FSHLogProvider(93): Error instantiating log writer. java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 27 more 2024-12-10T07:26:45,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,598 ERROR [LogRoller {}] wal.AbstractWALRoller(227): Roll wal failed and waiting timeout, will not retry java.io.IOException: cannot get log writer at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:95) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] ... 10 more Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] ... 10 more 2024-12-10T07:26:45,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,599 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T07:26:45,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T07:26:45,901 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1733815605590) 2024-12-10T07:26:45,901 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bfc47399-0870-848d-6457-efc0ac0affb2 2024-12-10T07:26:46,458 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 79) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4840 (was 5513)